zhicheng
/
caijitask
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

海外任务缓存

sitask
zhicheng.zhang 10 months ago
parent
27af7d0b5e
commit
f163c2acbc
930 changed files with 0 additions and 80838 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 7
      S/145redis.properties
  2. 18
      S/1_kafka.properties
  3. 18
      S/2_kafka.properties
  4. 18
      S/3_kafka.properties
  5. 18
      S/4_kafka.properties
  6. 18
      S/5_kafka.properties
  7. 5
      S/aadb.properties
  8. 10
      S/db.properties
  9. 10
      S/elasticsearch_1.properties
  10. 10
      S/elasticsearch_2.properties
  11. 21
      S/log4j.properties
  12. 36
      S/posTag.txt
  13. 7
      S/redis.properties
  14. 1280
      S/stopWords.txt
  15. 891
      S/stopWordsEN.txt
  16. 184
      cl_search_api/cl_search_api.iml
  17. 244
      cl_search_api/pom.xml
  18. 22
      cl_search_api/src/main/java/com/bfd/mf/SearchApplication.java
  19. 10
      cl_search_api/src/main/java/com/bfd/mf/common/service/ISubjectClusterService.java
  20. 65
      cl_search_api/src/main/java/com/bfd/mf/common/service/alexa/SourceAlexaManager.java
  21. 57
      cl_search_api/src/main/java/com/bfd/mf/common/service/base/GenericManager.java
  22. 44
      cl_search_api/src/main/java/com/bfd/mf/common/service/base/GenericTreeManager.java
  23. 61
      cl_search_api/src/main/java/com/bfd/mf/common/service/base/impl/GenericManagerImpl.java
  24. 64
      cl_search_api/src/main/java/com/bfd/mf/common/service/base/impl/GenericTreeManagerImpl.java
  25. 83
      cl_search_api/src/main/java/com/bfd/mf/common/service/cache/SubjectClusterService.java
  26. 120
      cl_search_api/src/main/java/com/bfd/mf/common/service/cache/TopicQueryService.java
  27. 183
      cl_search_api/src/main/java/com/bfd/mf/common/service/common/CrudService.java
  28. 34
      cl_search_api/src/main/java/com/bfd/mf/common/service/emotion/EmotionEntityService.java
  29. 33
      cl_search_api/src/main/java/com/bfd/mf/common/service/es/ClusterService.java
  30. 240
      cl_search_api/src/main/java/com/bfd/mf/common/service/es/ESCommonService.java
  31. 82
      cl_search_api/src/main/java/com/bfd/mf/common/service/es/MonitorMainSourceService.java
  32. 92
      cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java
  33. 266
      cl_search_api/src/main/java/com/bfd/mf/common/service/text/TextService.java
  34. 193
      cl_search_api/src/main/java/com/bfd/mf/common/thread/SubjectQueryDataService.java
  35. 1059
      cl_search_api/src/main/java/com/bfd/mf/common/util/ESConstant.java
  36. 118
      cl_search_api/src/main/java/com/bfd/mf/common/util/ESServerUtils.java
  37. 20
      cl_search_api/src/main/java/com/bfd/mf/common/util/EsQueryConditionUtils.java
  38. 64
      cl_search_api/src/main/java/com/bfd/mf/common/util/OperatorUtil.java
  39. 354
      cl_search_api/src/main/java/com/bfd/mf/common/util/analysis/DataAnalysisUtils.java
  40. 279
      cl_search_api/src/main/java/com/bfd/mf/common/util/analysis/DateTrendUtils.java
  41. 43
      cl_search_api/src/main/java/com/bfd/mf/common/util/cache/ResponseParseUtils.java
  42. 36
      cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ConditionCommon.java
  43. 663
      cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ESConstant.java
  44. 66
      cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ScoreFactorEntity.java
  45. 308
      cl_search_api/src/main/java/com/bfd/mf/common/util/enums/CacheEnums.java
  46. 78
      cl_search_api/src/main/java/com/bfd/mf/common/util/enums/DataExportStatus.java
  47. 119
      cl_search_api/src/main/java/com/bfd/mf/common/util/enums/RTCodeEnum.java
  48. 99
      cl_search_api/src/main/java/com/bfd/mf/common/util/enums/SpreadEnums.java
  49. 32
      cl_search_api/src/main/java/com/bfd/mf/common/util/es/MonitorConstant.java
  50. 52
      cl_search_api/src/main/java/com/bfd/mf/common/util/es/MonitorUtils.java
  51. 177
      cl_search_api/src/main/java/com/bfd/mf/common/util/slice/SliceScrollUtil.java
  52. 334
      cl_search_api/src/main/java/com/bfd/mf/common/util/spread/SpreadServiceUtil.java
  53. 1042
      cl_search_api/src/main/java/com/bfd/mf/common/util/subject/ExpNodeUtil.java
  54. 911
      cl_search_api/src/main/java/com/bfd/mf/common/util/subject/ExpressionParser.java
  55. 669
      cl_search_api/src/main/java/com/bfd/mf/common/util/subject/QueryBuilderUtil.java
  56. 110
      cl_search_api/src/main/java/com/bfd/mf/common/util/thread/SubjectDataQueryThread.java
  57. 123
      cl_search_api/src/main/java/com/bfd/mf/common/util/utility/CollectionUtils.java
  58. 292
      cl_search_api/src/main/java/com/bfd/mf/common/util/utility/DateUtil.java
  59. 18
      cl_search_api/src/main/java/com/bfd/mf/common/util/utility/EsQueryConditionUtils.java
  60. 518
      cl_search_api/src/main/java/com/bfd/mf/common/util/utility/TagWordUtils.java
  61. 72
      cl_search_api/src/main/java/com/bfd/mf/common/web/component/wrapper/ResponseWrapper.java
  62. 41
      cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/BookMark.java
  63. 64
      cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/Channel.java
  64. 50
      cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/Model.java
  65. 127
      cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/SentimentModify.java
  66. 49
      cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/SubjectModel.java
  67. 322
      cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/User.java
  68. 106
      cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/base/BaseEntityModel.java
  69. 34
      cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/base/BaseModel.java
  70. 113
      cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/base/BaseTreeEntityModel.java
  71. 166
      cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/cache/Cluster.java
  72. 114
      cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/cache/SubjectCluster.java
  73. 104
      cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/report/MainSourceEntity.java
  74. 425
      cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/topic/Subject.java
  75. 29
      cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/BookMarkRepository.java
  76. 13
      cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/SentimentRepository.java
  77. 469
      cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/UserRepository.java
  78. 13
      cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/base/GenericDao.java
  79. 13
      cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/base/GenericTreeDao.java
  80. 25
      cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/cache/ClusterRepository.java
  81. 12
      cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/cache/SubjectClusterRepository.java
  82. 14
      cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/report/MainSourceRepository.java
  83. 59
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/BaseRequest.java
  84. 259
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/QueryRequest.java
  85. 51
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/expression/DistanceNode.java
  86. 114
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/expression/ExpNode.java
  87. 108
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/expression/ExpObject.java
  88. 55
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/expression/HalfQuartChar.java
  89. 78
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/expression/OperaObject.java
  90. 249
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/post/PostMonitorRequest.java
  91. 54
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/spam/SpamConstant.java
  92. 16
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/womanalysis/ProductTag.java
  93. 94
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/DataCompareDocCount.java
  94. 37
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/DataCountBase.java
  95. 88
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/DataLineCount.java
  96. 97
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/DataPieCount.java
  97. 82
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/DataTrendCompare.java
  98. 90
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/DataTrendCount.java
  99. 73
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/NodeEmotionCount.java
  100. 52
      cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/PieCount.java

7
S/145redis.properties
View File

@ -1,7 +0,0 @@
#@author ruining.he
#redis.model -> install model {single,codis}
#if model is codis,use config as below
redis.model=single
redis.ip=172.18.1.145
redis.port=6379

18
S/1_kafka.properties
View File

@ -1,18 +0,0 @@
#### Kafka Tool Parameters
crawl.kfk.metadata.broker.list=172.18.1.119:9992
crawl.kfk.connectionTimeOut=50000
crawl.kfk.zk.sessiontimeout.ms=50000
crawl.kfk.kafka.serializer=kafka.serializer.StringEncoder
crawl.kfk.producer.acks=all
crawl.kfk.producer.retries=0
crawl.kfk.producer.batch.size=16384
crawl.kfk.producer.linger.ms=1
crawl.kfk.producer.buffer.memory=33554432
crawl.kfk.producer.max.request.size=10485760
crawl.kfk.consumer.enable.auto.commit=true
crawl.kfk.consumer.auto.commit.interval.ms=1000
crawl.kfk.consumer.session.timeout.ms=30000
crawl.kfk.consumer.auto.offset.reset=earliest
crawl.kfk.consumer.thread.core.pool.size=100
crawl.kfk.consumer.thread.maximum.pool.size=100

18
S/2_kafka.properties
View File

@ -1,18 +0,0 @@
#### Kafka Tool Parameters
crawl.kfk.metadata.broker.list=172.18.1.178:9092,172.18.1.181:9092,172.18.1.182:9092
crawl.kfk.connectionTimeOut=50000
crawl.kfk.zk.sessiontimeout.ms=50000
crawl.kfk.kafka.serializer=kafka.serializer.StringEncoder
crawl.kfk.producer.acks=all
crawl.kfk.producer.retries=0
crawl.kfk.producer.batch.size=16384
crawl.kfk.producer.linger.ms=1
crawl.kfk.producer.buffer.memory=33554432
crawl.kfk.producer.max.request.size=10485760
crawl.kfk.consumer.enable.auto.commit=true
crawl.kfk.consumer.auto.commit.interval.ms=1000
crawl.kfk.consumer.session.timeout.ms=30000
crawl.kfk.consumer.auto.offset.reset=latest
crawl.kfk.consumer.thread.core.pool.size=100
crawl.kfk.consumer.thread.maximum.pool.size=100

18
S/3_kafka.properties
View File

@ -1,18 +0,0 @@
#### Kafka Tool Parameters
crawl.kfk.metadata.broker.list=172.18.1.114:9092
crawl.kfk.connectionTimeOut=50000
crawl.kfk.zk.sessiontimeout.ms=50000
crawl.kfk.kafka.serializer=kafka.serializer.StringEncoder
crawl.kfk.producer.acks=all
crawl.kfk.producer.retries=0
crawl.kfk.producer.batch.size=16384
crawl.kfk.producer.linger.ms=1
crawl.kfk.producer.buffer.memory=33554432
crawl.kfk.producer.max.request.size=10485760
crawl.kfk.consumer.enable.auto.commit=true
crawl.kfk.consumer.auto.commit.interval.ms=1000
crawl.kfk.consumer.session.timeout.ms=30000
crawl.kfk.consumer.auto.offset.reset=earliest
crawl.kfk.consumer.thread.core.pool.size=100
crawl.kfk.consumer.thread.maximum.pool.size=100

18
S/4_kafka.properties
View File

@ -1,18 +0,0 @@
#### Kafka Tool Parameters
crawl.kfk.metadata.broker.list=50.19.49.16:9092
crawl.kfk.connectionTimeOut=50000
crawl.kfk.zk.sessiontimeout.ms=50000
crawl.kfk.kafka.serializer=kafka.serializer.StringEncoder
crawl.kfk.producer.acks=all
crawl.kfk.producer.retries=0
crawl.kfk.producer.batch.size=16384
crawl.kfk.producer.linger.ms=1
crawl.kfk.producer.buffer.memory=33554432
crawl.kfk.producer.max.request.size=10485760
crawl.kfk.consumer.enable.auto.commit=true
crawl.kfk.consumer.auto.commit.interval.ms=1000
crawl.kfk.consumer.session.timeout.ms=30000
crawl.kfk.consumer.auto.offset.reset=earliest
crawl.kfk.consumer.thread.core.pool.size=100
crawl.kfk.consumer.thread.maximum.pool.size=100

18
S/5_kafka.properties
View File

@ -1,18 +0,0 @@
#### Kafka Tool Parameters
crawl.kfk.metadata.broker.list=172.18.1.113:9092
crawl.kfk.connectionTimeOut=50000
crawl.kfk.zk.sessiontimeout.ms=50000
crawl.kfk.kafka.serializer=kafka.serializer.StringEncoder
crawl.kfk.producer.acks=all
crawl.kfk.producer.retries=0
crawl.kfk.producer.batch.size=16384
crawl.kfk.producer.linger.ms=1
crawl.kfk.producer.buffer.memory=33554432
crawl.kfk.producer.max.request.size=10485760
crawl.kfk.consumer.enable.auto.commit=true
crawl.kfk.consumer.auto.commit.interval.ms=1000
crawl.kfk.consumer.session.timeout.ms=30000
crawl.kfk.consumer.auto.offset.reset=earliest
crawl.kfk.consumer.thread.core.pool.size=100
crawl.kfk.consumer.thread.maximum.pool.size=100

5
S/aadb.properties
View File

@ -1,5 +0,0 @@
logfile=DBConnectManager.log
db_stat_alltask.maxconn=100
db_stat_alltask.url=jdbc:mysql://172.18.1.134:3306/all_task?useUnicode=true&characterEncoding=utf-8
db_stat_alltask.user=root
db_stat_alltask.password=Bfd123!@#

10
S/db.properties
View File

@ -1,10 +0,0 @@
logfile=DBConnectManager.log
drivers=com.mysql.jdbc.Driver
db_stat.maxconn=100
db_stat.url=jdbc:mysql://172.18.1.101:3306/intelligent_crawl?useUnicode=true&characterEncoding=utf-8
db_stat.user=crawl
db_stat.password=crawl123
db_stat_alltask.maxconn=100
db_stat_alltask.url=jdbc:mysql://172.18.1.101:3306/all_task?useUnicode=true&characterEncoding=utf-8
db_stat_alltask.user=crawl
db_stat_alltask.password=crawl123=

10
S/elasticsearch_1.properties
View File

@ -1,10 +0,0 @@
crawl.elasticsearch.metadata.broker.list=172.26.11.111:9201
crawl.elasticsearch.socket.timeout.interval.ms=5000
crawl.elasticsearch.connect.timeout.ms=1000
crawl.elasticsearch.max.connect.num=100
crawl.elasticsearch.max.connect.per.route.num=100
crawl.elasticsearch.max.retry.timeout=300000
crawl.elasticsearch.producer.max.cache.num=2000
crawl.elasticsearch.producer.max.write.time=120000
crawl.elasticsearch.consumer.quary.num=2000
crawl.elasticsearch.consumer.scroll.time=3

10
S/elasticsearch_2.properties
View File

@ -1,10 +0,0 @@
crawl.elasticsearch.metadata.broker.list=172.26.11.109:9201
crawl.elasticsearch.socket.timeout.interval.ms=60000
crawl.elasticsearch.connect.timeout.ms=50000
crawl.elasticsearch.max.connect.num=100
crawl.elasticsearch.max.connect.per.route.num=100
crawl.elasticsearch.max.retry.timeout=300000
crawl.elasticsearch.producer.max.cache.num=1000
crawl.elasticsearch.producer.max.write.time=100000
crawl.elasticsearch.consumer.quary.num=1000
crawl.elasticsearch.consumer.scroll.time=3

21
S/log4j.properties
View File

@ -1,21 +0,0 @@
log4j.rootLogger=debug
log4j.logger.com.bfd.mf.service = DEBUG,SERVICE
log4j.logger.com.bfd.mf.datasave = DEBUG,DATASAVE
#service log
log4j.appender.SERVICE=org.apache.log4j.DailyRollingFileAppender
log4j.appender.SERVICE.Threshold=DEBUG
log4j.appender.SERVICE.File=../logs/service/service.log
log4j.appender.SERVICE.DatePattern='.'yyyy-MM-dd
log4j.appender.SERVICE.layout=org.apache.log4j.PatternLayout
log4j.appender.SERVICE.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%L) - %m%n
log4j.logger.com.bfd.mf.datasave = DEBUG,DATASAVE
#datasave log
log4j.appender.DATASAVE=org.apache.log4j.DailyRollingFileAppender
log4j.appender.DATASAVE.Threshold=DEBUG
log4j.appender.DATASAVE.File=../logs/datasave/datasave.log
log4j.appender.DATASAVE.DatePattern='.'yyyy-MM-dd
log4j.appender.DATASAVE.layout=org.apache.log4j.PatternLayout
log4j.appender.DATASAVE.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%L) - %m%n

36
S/posTag.txt
View File

@ -1,36 +0,0 @@
u
ud
ude1
ude2
ude3
udeng
udh
ug
uguo
uj
ul
ule
ulian
uls
usuo
uv
uyy
uz
uzhe
uzhi
w
wb
wd
wf
wh
wj
wky
wkz
wm
wn
wp
ws
wt
ww
wyy
wyz

7
S/redis.properties
View File

@ -1,7 +0,0 @@
#@author ruining.he
#redis.model -> install model {single,codis}
#if model is codis,use config as below
redis.model=single
redis.ip=172.18.1.152
redis.port=6379

1280
S/stopWords.txt
File diff suppressed because it is too large
View File

891
S/stopWordsEN.txt
View File

@ -1,891 +0,0 @@
'd
'll
'm
're
's
't
've
ZT
ZZ
a
a's
able
about
above
abst
accordance
according
accordingly
across
act
actually
added
adj
adopted
affected
affecting
affects
after
afterwards
again
against
ah
ain't
all
allow
allows
almost
alone
along
already
also
although
always
am
among
amongst
an
and
announce
another
any
anybody
anyhow
anymore
anyone
anything
anyway
anyways
anywhere
apart
apparently
appear
appreciate
appropriate
approximately
are
area
areas
aren
aren't
arent
arise
around
as
aside
ask
asked
asking
asks
associated
at
auth
available
away
awfully
b
back
backed
backing
backs
be
became
because
become
becomes
becoming
been
before
beforehand
began
begin
beginning
beginnings
begins
behind
being
beings
believe
below
beside
besides
best
better
between
beyond
big
biol
both
brief
briefly
but
by
c
c'mon
c's
ca
came
can
can't
cannot
cant
case
cases
cause
causes
certain
certainly
changes
clear
clearly
co
com
come
comes
concerning
consequently
consider
considering
contain
containing
contains
corresponding
could
couldn't
couldnt
course
currently
d
date
definitely
describe
described
despite
did
didn't
differ
different
differently
discuss
do
does
doesn't
doing
don't
done
down
downed
downing
downs
downwards
due
during
e
each
early
ed
edu
effect
eg
eight
eighty
either
else
elsewhere
end
ended
ending
ends
enough
entirely
especially
et
et-al
etc
even
evenly
ever
every
everybody
everyone
everything
everywhere
ex
exactly
example
except
f
face
faces
fact
facts
far
felt
few
ff
fifth
find
finds
first
five
fix
followed
following
follows
for
former
formerly
forth
found
four
from
full
fully
further
furthered
furthering
furthermore
furthers
g
gave
general
generally
get
gets
getting
give
given
gives
giving
go
goes
going
gone
good
goods
got
gotten
great
greater
greatest
greetings
group
grouped
grouping
groups
h
had
hadn't
happens
hardly
has
hasn't
have
haven't
having
he
he's
hed
hello
help
hence
her
here
here's
hereafter
hereby
herein
heres
hereupon
hers
herself
hes
hi
hid
high
higher
highest
him
himself
his
hither
home
hopefully
how
howbeit
however
hundred
i
i'd
i'll
i'm
i've
id
ie
if
ignored
im
immediate
immediately
importance
important
in
inasmuch
inc
include
indeed
index
indicate
indicated
indicates
information
inner
insofar
instead
interest
interested
interesting
interests
into
invention
inward
is
isn't
it
it'd
it'll
it's
itd
its
itself
j
just
k
keep
keeps
kept
keys
kg
kind
km
knew
know
known
knows
l
large
largely
last
lately
later
latest
latter
latterly
least
less
lest
let
let's
lets
like
liked
likely
line
little
long
longer
longest
look
looking
looks
ltd
m
made
mainly
make
makes
making
man
many
may
maybe
me
mean
means
meantime
meanwhile
member
members
men
merely
mg
might
million
miss
ml
more
moreover
most
mostly
mr
mrs
much
mug
must
my
myself
n
n't
na
name
namely
nay
nd
near
nearly
necessarily
necessary
need
needed
needing
needs
neither
never
nevertheless
new
newer
newest
next
nine
ninety
no
nobody
non
none
nonetheless
noone
nor
normally
nos
not
noted
nothing
novel
now
nowhere
number
numbers
o
obtain
obtained
obviously
of
off
often
oh
ok
okay
old
older
oldest
omitted
on
once
one
ones
only
onto
open
opened
opening
opens
or
ord
order
ordered
ordering
orders
other
others
otherwise
ought
our
ours
ourselves
out
outside
over
overall
owing
own
p
page
pages
part
parted
particular
particularly
parting
parts
past
per
perhaps
place
placed
places
please
plus
point
pointed
pointing
points
poorly
possible
possibly
potentially
pp
predominantly
present
presented
presenting
presents
presumably
previously
primarily
probably
problem
problems
promptly
proud
provides
put
puts
q
que
quickly
quite
qv
r
ran
rather
rd
re
readily
really
reasonably
recent
recently
ref
refs
regarding
regardless
regards
related
relatively
research
respectively
resulted
resulting
results
right
room
rooms
run
s
said
same
saw
say
saying
says
sec
second
secondly
seconds
section
see
seeing
seem
seemed
seeming
seems
seen
sees
self
selves
sensible
sent
serious
seriously
seven
several
shall
she
she'll
shed
shes
should
shouldn't
show
showed
showing
shown
showns
shows
side
sides
significant
significantly
similar
similarly
since
six
slightly
small
smaller
smallest
so
some
somebody
somehow
someone
somethan
something
sometime
sometimes
somewhat
somewhere
soon
sorry
specifically
specified
specify
specifying
state
states
still
stop
strongly
sub
substantially
successfully
such
sufficiently
suggest
sup
sure
t
t's
take
taken
taking
tell
tends
th
than
thank
thanks
thanx
that
that'll
that's
that've
thats
the
their
theirs
them
themselves
then
thence
there
there'll
there's
there've
thereafter
thereby
thered
therefore
therein
thereof
therere
theres
thereto
thereupon
these
they
they'd
they'll
they're
they've
theyd
theyre
thing
things
think
thinks
third
this
thorough
thoroughly
those
thou
though
thoughh
thought
thoughts
thousand
three
throug
through
throughout
thru
thus
til
tip
to
today
together
too
took
toward
towards
tried
tries
truly
try
trying
ts
turn
turned
turning
turns
twice
two
u
un
under
unfortunately
unless
unlike
unlikely
until
unto
up
upon
ups
us
use
used
useful
usefully
usefulness
uses
using
usually
uucp
v
value
various
very
via
viz
vol
vols
vs
w
want
wanted
wanting
wants
was
wasn't
way
ways
we
we'd
we'll
we're
we've
wed
welcome
well
wells
went
were
weren't
what
what'll
what's
whatever
whats
when
whence
whenever
where
where's
whereafter
whereas
whereby
wherein
wheres
whereupon
wherever
whether
which
while
whim
whither
who
who'll
who's
whod
whoever
whole
whom
whomever
whos
whose
why
widely
will
willing
wish
with
within
without
won't
wonder
words
work
worked
working
works
world
would
wouldn't
www
x
y
year
years
yes
yet
you
you'd
you'll
you're
you've
youd
young
younger
youngest
your
youre
yours
yourself
yourselves
z
zero
zt
zz

184
cl_search_api/cl_search_api.iml
View File

@ -1,184 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
<component name="FacetManager">
<facet type="web" name="Web">
<configuration>
<webroots />
</configuration>
</facet>
<facet type="jpa" name="JPA">
<configuration>
<setting name="validation-enabled" value="true" />
<setting name="provider-name" value="Hibernate" />
<datasource-mapping>
<factory-entry name="entityManagerFactory" />
</datasource-mapping>
<naming-strategy-map />
</configuration>
</facet>
<facet type="Spring" name="Spring">
<configuration />
</facet>
</component>
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
<output url="file://$MODULE_DIR$/target/classes" />
<output-test url="file://$MODULE_DIR$/target/test-classes" />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/main/resources" type="java-resource" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Maven: jdk.tools:jdk.tools:1.8" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-webmvc:5.0.4.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-aop:5.0.4.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-beans:5.0.4.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-context:5.0.4.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-expression:5.0.4.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-web:5.0.4.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-core:5.0.4.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-jcl:5.0.4.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-web:2.0.0.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter:2.0.0.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-logging:2.0.0.RELEASE" level="project" />
<orderEntry type="library" name="Maven: ch.qos.logback:logback-classic:1.2.3" level="project" />
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-to-slf4j:2.10.0" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:jul-to-slf4j:1.7.25" level="project" />
<orderEntry type="library" name="Maven: javax.annotation:javax.annotation-api:1.3.2" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-tomcat:2.0.0.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.apache.tomcat.embed:tomcat-embed-core:8.5.28" level="project" />
<orderEntry type="library" name="Maven: org.apache.tomcat.embed:tomcat-embed-el:8.5.28" level="project" />
<orderEntry type="library" name="Maven: org.apache.tomcat.embed:tomcat-embed-websocket:8.5.28" level="project" />
<orderEntry type="library" name="Maven: org.hibernate.validator:hibernate-validator:6.0.7.Final" level="project" />
<orderEntry type="library" name="Maven: javax.validation:validation-api:2.0.1.Final" level="project" />
<orderEntry type="library" name="Maven: org.jboss.logging:jboss-logging:3.3.0.Final" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-test:2.0.0.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-test:2.0.0.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-test-autoconfigure:2.0.0.RELEASE" level="project" />
<orderEntry type="library" name="Maven: com.jayway.jsonpath:json-path:2.4.0" level="project" />
<orderEntry type="library" name="Maven: net.minidev:json-smart:2.3" level="project" />
<orderEntry type="library" name="Maven: net.minidev:accessors-smart:1.2" level="project" />
<orderEntry type="library" name="Maven: org.ow2.asm:asm:5.0.4" level="project" />
<orderEntry type="library" name="Maven: junit:junit:4.12" level="project" />
<orderEntry type="library" name="Maven: org.assertj:assertj-core:3.9.1" level="project" />
<orderEntry type="library" name="Maven: org.mockito:mockito-core:2.15.0" level="project" />
<orderEntry type="library" name="Maven: net.bytebuddy:byte-buddy:1.7.9" level="project" />
<orderEntry type="library" name="Maven: net.bytebuddy:byte-buddy-agent:1.7.9" level="project" />
<orderEntry type="library" name="Maven: org.objenesis:objenesis:2.6" level="project" />
<orderEntry type="library" name="Maven: org.hamcrest:hamcrest-core:1.3" level="project" />
<orderEntry type="library" name="Maven: org.hamcrest:hamcrest-library:1.3" level="project" />
<orderEntry type="library" name="Maven: org.skyscreamer:jsonassert:1.5.0" level="project" />
<orderEntry type="library" name="Maven: com.vaadin.external.google:android-json:0.0.20131108.vaadin1" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-test:5.0.4.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.xmlunit:xmlunit-core:2.5.1" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-data-jpa:2.0.0.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-aop:2.0.0.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.aspectj:aspectjweaver:1.8.13" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-starter-jdbc:2.0.0.RELEASE" level="project" />
<orderEntry type="library" name="Maven: com.zaxxer:HikariCP:2.7.8" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-jdbc:5.0.4.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.hibernate:hibernate-core:5.2.14.Final" level="project" />
<orderEntry type="library" name="Maven: org.hibernate.javax.persistence:hibernate-jpa-2.1-api:1.0.0.Final" level="project" />
<orderEntry type="library" name="Maven: org.javassist:javassist:3.22.0-GA" level="project" />
<orderEntry type="library" name="Maven: antlr:antlr:2.7.7" level="project" />
<orderEntry type="library" name="Maven: org.jboss:jandex:2.0.3.Final" level="project" />
<orderEntry type="library" name="Maven: dom4j:dom4j:1.6.1" level="project" />
<orderEntry type="library" name="Maven: org.hibernate.common:hibernate-commons-annotations:5.0.1.Final" level="project" />
<orderEntry type="library" name="Maven: javax.transaction:javax.transaction-api:1.2" level="project" />
<orderEntry type="library" name="Maven: org.springframework.data:spring-data-jpa:2.0.5.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework.data:spring-data-commons:2.0.5.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-orm:5.0.4.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-tx:5.0.4.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework:spring-aspects:5.0.4.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-devtools:2.0.0.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot:2.0.0.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework.boot:spring-boot-autoconfigure:2.0.0.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.mybatis.spring.boot:mybatis-spring-boot-starter:1.3.1" level="project" />
<orderEntry type="library" name="Maven: org.mybatis.spring.boot:mybatis-spring-boot-autoconfigure:1.3.1" level="project" />
<orderEntry type="library" name="Maven: org.mybatis:mybatis:3.4.5" level="project" />
<orderEntry type="library" name="Maven: org.mybatis:mybatis-spring:1.3.1" level="project" />
<orderEntry type="library" name="Maven: org.elasticsearch:elasticsearch:6.0.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-core:7.0.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-analyzers-common:7.0.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-backward-codecs:7.0.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-grouping:7.0.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-highlighter:7.0.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-join:7.0.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-memory:7.0.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-misc:7.0.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-queries:7.0.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-queryparser:7.0.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-sandbox:7.0.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-spatial:7.0.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-spatial-extras:7.0.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-spatial3d:7.0.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.lucene:lucene-suggest:7.0.1" level="project" />
<orderEntry type="library" name="Maven: org.elasticsearch:securesm:1.1" level="project" />
<orderEntry type="library" name="Maven: net.sf.jopt-simple:jopt-simple:5.0.2" level="project" />
<orderEntry type="library" name="Maven: com.carrotsearch:hppc:0.7.1" level="project" />
<orderEntry type="library" name="Maven: joda-time:joda-time:2.9.5" level="project" />
<orderEntry type="library" name="Maven: org.yaml:snakeyaml:1.15" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-core:2.8.6" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.8.6" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.8.6" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.8.6" level="project" />
<orderEntry type="library" name="Maven: com.tdunning:t-digest:3.0" level="project" />
<orderEntry type="library" name="Maven: org.hdrhistogram:HdrHistogram:2.1.9" level="project" />
<orderEntry type="library" name="Maven: org.apache.logging.log4j:log4j-api:2.9.1" level="project" />
<orderEntry type="library" name="Maven: org.elasticsearch:jna:4.4.0-1" level="project" />
<orderEntry type="library" name="Maven: org.elasticsearch.client:transport:6.0.0" level="project" />
<orderEntry type="library" name="Maven: org.elasticsearch.plugin:transport-netty4-client:6.0.0" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-buffer:4.1.13.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec:4.1.13.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-codec-http:4.1.13.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-common:4.1.13.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-handler:4.1.13.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-resolver:4.1.13.Final" level="project" />
<orderEntry type="library" name="Maven: io.netty:netty-transport:4.1.13.Final" level="project" />
<orderEntry type="library" name="Maven: org.elasticsearch.plugin:reindex-client:6.0.0" level="project" />
<orderEntry type="library" name="Maven: org.elasticsearch.client:elasticsearch-rest-client:6.0.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpclient:4.5.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore:4.4.5" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpasyncclient:4.1.2" level="project" />
<orderEntry type="library" name="Maven: org.apache.httpcomponents:httpcore-nio:4.4.5" level="project" />
<orderEntry type="library" name="Maven: commons-logging:commons-logging:1.1.1" level="project" />
<orderEntry type="library" name="Maven: org.elasticsearch.plugin:lang-mustache-client:6.0.0" level="project" />
<orderEntry type="library" name="Maven: com.github.spullara.mustache.java:compiler:0.9.3" level="project" />
<orderEntry type="library" name="Maven: org.elasticsearch.plugin:percolator-client:6.0.0" level="project" />
<orderEntry type="library" name="Maven: org.elasticsearch.plugin:parent-join-client:6.0.0" level="project" />
<orderEntry type="library" name="Maven: org.apache.poi:poi:3.15" level="project" />
<orderEntry type="library" name="Maven: commons-codec:commons-codec:1.10" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-collections4:4.1" level="project" />
<orderEntry type="library" name="Maven: org.apache.poi:poi-ooxml:3.15" level="project" />
<orderEntry type="library" name="Maven: org.apache.poi:poi-ooxml-schemas:3.15" level="project" />
<orderEntry type="library" name="Maven: org.apache.xmlbeans:xmlbeans:2.6.0" level="project" />
<orderEntry type="library" name="Maven: stax:stax-api:1.0.1" level="project" />
<orderEntry type="library" name="Maven: com.github.virtuald:curvesapi:1.04" level="project" />
<orderEntry type="library" name="Maven: com.alibaba:fastjson:1.1.22" level="project" />
<orderEntry type="library" name="Maven: org.projectlombok:lombok:1.16.20" level="project" />
<orderEntry type="library" name="Maven: com.google.guava:guava:19.0" level="project" />
<orderEntry type="library" name="Maven: mysql:mysql-connector-java:5.1.29" level="project" />
<orderEntry type="library" name="Maven: org.jodd:jodd-core:3.4.8" level="project" />
<orderEntry type="library" name="Maven: com.swagger.ui:swagger-bootstrap-ui:1.8.8" level="project" />
<orderEntry type="library" name="Maven: io.springfox:springfox-swagger2:2.6.1" level="project" />
<orderEntry type="library" name="Maven: io.swagger:swagger-annotations:1.5.10" level="project" />
<orderEntry type="library" name="Maven: io.swagger:swagger-models:1.5.10" level="project" />
<orderEntry type="library" name="Maven: io.springfox:springfox-spi:2.6.1" level="project" />
<orderEntry type="library" name="Maven: io.springfox:springfox-core:2.6.1" level="project" />
<orderEntry type="library" name="Maven: io.springfox:springfox-schema:2.6.1" level="project" />
<orderEntry type="library" name="Maven: io.springfox:springfox-swagger-common:2.6.1" level="project" />
<orderEntry type="library" name="Maven: io.springfox:springfox-spring-web:2.6.1" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml:classmate:1.3.1" level="project" />
<orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.21" level="project" />
<orderEntry type="library" name="Maven: org.springframework.plugin:spring-plugin-core:1.2.0.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.springframework.plugin:spring-plugin-metadata:1.2.0.RELEASE" level="project" />
<orderEntry type="library" name="Maven: org.mapstruct:mapstruct:1.0.0.Final" level="project" />
<orderEntry type="library" name="Maven: org.apache.commons:commons-lang3:3.2.1" level="project" />
<orderEntry type="library" name="Maven: com.bfd.nlp:nlp_common_util:1.1" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.9.5" level="project" />
<orderEntry type="library" name="Maven: com.fasterxml.jackson.core:jackson-annotations:2.9.0" level="project" />
<orderEntry type="library" name="Maven: net.logstash.logback:logstash-logback-encoder:4.4" level="project" />
<orderEntry type="library" name="Maven: ch.qos.logback:logback-core:1.1.3" level="project" />
</component>
</module>

244
cl_search_api/pom.xml
View File

@ -1,244 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>cl_stream</artifactId>
<groupId>com.bfd.mf</groupId>
<version>2.0-SNAPSHOT</version>
</parent>
<name>cl_search_api</name>
<description>Search V2.0 API</description>
<artifactId>cl_search_api</artifactId>
<version>2.0-SNAPSHOT</version>
<properties>
<start-class>com.bfd.mf.SearchApplication</start-class>
<source>1.8</source>
<es.version>6.0.0</es.version>
<spring-boot-version>2.0.0.RELEASE</spring-boot-version>
<springframework.boot.version>2.0.0.RELEASE</springframework.boot.version>
<springframework.version>5.0.4.RELEASE</springframework.version>
<logstash.version>4.4</logstash.version>
<jna.version>4.1.0</jna.version>
<jetty.version>9.4.8.v20171121</jetty.version>
<druid.version>1.1.6</druid.version>
<guava.version>19.0</guava.version>
<poi.version>3.15</poi.version>
<java.version>1.8</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
</properties>
<dependencies>
<dependency>
<groupId>jdk.tools</groupId>
<artifactId>jdk.tools</artifactId>
<version>1.8</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-webmvc</artifactId>
<version>${springframework.version}</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-core</artifactId>
<version>${springframework.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<version>${springframework.boot.version}</version>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-json</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework</groupId>
<artifactId>spring-web</artifactId>
</exclusion>
<exclusion>
<groupId>org.springframework</groupId>
<artifactId>spring-webmvc</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<version>${springframework.boot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-jpa</artifactId>
<version>${springframework.boot.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-devtools</artifactId>
<version>${springframework.boot.version}</version>
<!--optional=true,依赖不会传递，该项目依赖devtools；之后依赖该项目的项目如果想要使用devtools，需要重新引入 -->
<optional>true</optional>
<!--<scope>true</scope>-->
</dependency>
<dependency>
<groupId>org.mybatis.spring.boot</groupId>
<artifactId>mybatis-spring-boot-starter</artifactId>
<version>1.3.1</version>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>${es.version}</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>transport</artifactId>
<version>${es.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>${poi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>${poi.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.1.22</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.16.20</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>19.0</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.29</version>
</dependency>
<dependency>
<groupId>org.jodd</groupId>
<artifactId>jodd-core</artifactId>
<version>3.4.8</version>
</dependency>
<dependency>
<groupId>com.swagger.ui</groupId>
<artifactId>swagger-bootstrap-ui</artifactId>
<version>1.8.8</version>
</dependency>
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger2</artifactId>
<version>2.6.1</version>
<exclusions>
<exclusion>
<artifactId>guava</artifactId>
<groupId>com.google.guava</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.2.1</version>
</dependency>
<dependency>
<groupId>com.bfd.nlp</groupId>
<artifactId>nlp_common_util</artifactId>
<version>1.1</version>
<exclusions>
<exclusion>
<artifactId>httpclient</artifactId>
<groupId>org.apache.httpcomponents</groupId>
</exclusion>
</exclusions>
</dependency>
<!-- 解决构建失败问题 -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.9.5</version>
</dependency>
<!--<dependency>-->
<!--<groupId>com.fasterxml.jackson.core</groupId>-->
<!--<artifactId>jackson-core</artifactId>-->
<!--<version>2.9.6</version>-->
<!--</dependency>-->
<!--<dependency>-->
<!--<groupId>com.fasterxml.jackson.core</groupId>-->
<!--<artifactId>jackson-annotations</artifactId>-->
<!--<version>2.9.6</version>-->
<!--</dependency>-->
<!--<dependency>-->
<!--<groupId>com.fasterxml.jackson.module</groupId>-->
<!--<artifactId>jackson-module-jaxb-annotations</artifactId>-->
<!--<version>2.9.6</version>-->
<!--</dependency>-->
<dependency>
<groupId>net.logstash.logback</groupId>
<artifactId>logstash-logback-encoder</artifactId>
<version>4.4</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<mainClass>com.bfd.mf.SearchApplication</mainClass>
</configuration>
<executions>
<execution>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

22
cl_search_api/src/main/java/com/bfd/mf/SearchApplication.java
View File

@ -1,22 +0,0 @@
package com.bfd.mf;
import org.mybatis.spring.annotation.MapperScan;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import java.io.IOException;
@SpringBootApplication
@MapperScan("com.bfd.mf")
public class SearchApplication {
private static final Logger LOGGER = LoggerFactory.getLogger(SearchApplication.class);
public static void main(String[] args) throws IOException {
LOGGER.info("[SearchApplication] Start SearchApplication REST API...");
SpringApplication.run(SearchApplication.class, args);
LOGGER.info("[SearchApplication] Start SearchApplication REST API finish...");
}
}

10
cl_search_api/src/main/java/com/bfd/mf/common/service/ISubjectClusterService.java
View File

@ -1,10 +0,0 @@
package com.bfd.mf.common.service;
public interface ISubjectClusterService {
int createIndexBySubject(Long companyId,Long subjectId);
boolean deleteIndexBySubject(Long companyId,Long subjectId);
}

65
cl_search_api/src/main/java/com/bfd/mf/common/service/alexa/SourceAlexaManager.java
View File

@ -1,65 +0,0 @@
//package com.bfd.mf.common.service.alexa;
//
//import com.bfd.mf.common.service.base.impl.GenericManagerImpl;
//import com.bfd.mf.common.web.entity.mysql.SourceAlexa;
//import com.bfd.mf.common.web.repository.mysql.SourceAlexaRepository;
//import com.bfd.nlp.common.util.string.TStringUtils;
//import org.slf4j.Logger;
//import org.slf4j.LoggerFactory;
//import org.springframework.beans.factory.annotation.Autowired;
//import org.springframework.stereotype.Service;
//
//import java.util.HashMap;
//import java.util.List;
//import java.util.Map;
//
//@Service
//public class SourceAlexaManager extends GenericManagerImpl<SourceAlexa, Long> {
// private static final Logger logger = LoggerFactory.getLogger(SourceAlexaManager.class);
// SourceAlexaRepository sourceAlexaRepository;
// @Autowired
// public void setSourceAlexaRepository(SourceAlexaRepository sourceAlexaRepository) {
// this.sourceAlexaRepository = sourceAlexaRepository;
// this.dao = this.sourceAlexaRepository;
// }
//
// /**
// * 通过网站名称来匹配
// * @return
// */
// public Map<String,Float> findAllSourceAlexaBySiteName(){
// Map<String,Float> scoreMap = new HashMap();
// List<SourceAlexa> sourceAlexaList = sourceAlexaRepository.findAll();
// for (SourceAlexa sourceAlexa:sourceAlexaList) {
// if(TStringUtils.isNotEmpty(sourceAlexa.getSiteName())){
// /**
// xinhuanet.com 新华网
// xinhua.org 新华网
// 只需要保留靠前的域名
// */
// if (scoreMap.containsKey(sourceAlexa.getSiteName()) && scoreMap.get(sourceAlexa.getSiteName()) < sourceAlexa.getFinalScore())
// continue;
// else
// scoreMap.put(sourceAlexa.getSiteName(),sourceAlexa.getFinalScore());
//
//
// }
// }
// return scoreMap;
// }
//
// /**
// *通过Host来匹配
// * @return
// */
// public Map<String,Float> findAllSourceAlexaByHost(){
// Map<String,Float> scoreMap = new HashMap();
// List<SourceAlexa> sourceAlexaList = sourceAlexaRepository.findAll();
// for (SourceAlexa sourceAlexa:sourceAlexaList) {
// if(TStringUtils.isNotEmpty(sourceAlexa.getHost())){
// scoreMap.put(sourceAlexa.getHost(),sourceAlexa.getFinalScore());
// }
// }
// return scoreMap;
// }
//}

57
cl_search_api/src/main/java/com/bfd/mf/common/service/base/GenericManager.java
View File

@ -1,57 +0,0 @@
package com.bfd.mf.common.service.base;
import com.bfd.mf.common.web.entity.mysql.base.BaseEntityModel;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import java.io.Serializable;
import java.util.List;
public interface GenericManager<T extends BaseEntityModel, PK extends Serializable> {
/**
*
* @param page
* @return
*/
public Page<T> findAll(Pageable page);
/**
* fetch all entities
*
* @return list for all entities
*/
public List<T> findAll();
/**
* fetch specified entity according id;
*
* @param id
* entity id;
* @return entity
*/
public T findById(PK id);
/**
* save specified entity;
*
* @param entity
* entity for saving
* @return saved domain entity
*/
public T save(T entity);
/**
*
* @param entities
* @return
*/
public List<T> save(Iterable<T> entities);
/**
* * delete entity according given id
*
* @param id
*/
public void delete(PK id);
}

44
cl_search_api/src/main/java/com/bfd/mf/common/service/base/GenericTreeManager.java
View File

@ -1,44 +0,0 @@
package com.bfd.mf.common.service.base;
import com.bfd.mf.common.web.entity.mysql.base.BaseTreeEntityModel;
import java.io.Serializable;
import java.util.List;
@SuppressWarnings("rawtypes")
public interface GenericTreeManager<T extends BaseTreeEntityModel, PK extends Serializable>
extends GenericManager<T, PK> {
/**
* 获取根节点实体集合
*
* @return
*/
public List<T> getRoot();
/**
* 获取制定id的直接孩子实体集合；
*
* @param id
* @return
*/
public List<T> getChildren(PK id);
/**
* 获取制定id的全部子孙实体集合；
*
* @param id
* @return
*/
public List<T> getDescendants(PK id);
/**
* 获取制定id的全部祖先实体集合；
*
* @param id
* @return
*/
public List<T> getAncestors(PK id);
}

61
cl_search_api/src/main/java/com/bfd/mf/common/service/base/impl/GenericManagerImpl.java
View File

@ -1,61 +0,0 @@
package com.bfd.mf.common.service.base.impl;
import com.bfd.mf.common.service.base.GenericManager;
import com.bfd.mf.common.web.entity.mysql.base.BaseEntityModel;
import com.bfd.mf.common.web.repository.mysql.base.GenericDao;
import org.springframework.data.domain.Page;
import org.springframework.data.domain.Pageable;
import org.springframework.transaction.annotation.Transactional;
import java.io.Serializable;
import java.util.Date;
import java.util.List;
import java.util.Optional;
@Transactional
public class GenericManagerImpl<T extends BaseEntityModel, PK extends Serializable>
implements GenericManager<T, PK> {
protected GenericDao<T, PK> dao;
@Override
public Page<T> findAll(Pageable page) {
Page<T> result = this.dao.findAll(page);
return result;
}
@Override
public T findById(PK id) {
Optional<T> optional = this.dao.findById(id);
return optional.orElse(null);
}
@Override
public T save(T entity) {
Date date = new Date();
Long id = entity.getId();
if (id == null) {
entity.setDateCreated(date);
entity.setDateModified(date);
} else {
entity.setDateModified(date);
}
return this.dao.saveAndFlush(entity);
}
@Override
public List<T> save(Iterable<T> entities) {
return this.dao.saveAll(entities);
}
@Override
public List<T> findAll() {
return this.dao.findAll();
}
@Override
public void delete(PK id) {
this.dao.deleteById(id);
}
}

64
cl_search_api/src/main/java/com/bfd/mf/common/service/base/impl/GenericTreeManagerImpl.java
View File

@ -1,64 +0,0 @@
package com.bfd.mf.common.service.base.impl;
import com.bfd.mf.common.service.base.GenericTreeManager;
import com.bfd.mf.common.web.entity.mysql.base.BaseTreeEntityModel;
import com.bfd.mf.common.web.repository.mysql.base.GenericTreeDao;
import org.springframework.transaction.annotation.Transactional;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
@Transactional
public class GenericTreeManagerImpl<T extends BaseTreeEntityModel<T>, PK extends Serializable>
extends GenericManagerImpl<T, PK> implements GenericTreeManager<T, PK> {
protected GenericTreeDao<T, PK> treeDao;
@Override
public List<T> getRoot() {
return this.treeDao.getRoot();
}
@Override
public List<T> getChildren(PK id) {
T entity = this.treeDao.getOne(id);
return entity.getChildren();
}
@Override
public List<T> getDescendants(PK id) {
List<T> descendants = new ArrayList<T>();
List<T> children = this.getChildren(id);
if (children != null && children.size() > 0) { // 追加子孙
descendants.addAll(children);
for (T t : children) {
@SuppressWarnings("unchecked")
List<T> descendants2 = this.getDescendants((PK) t.getId());
descendants.addAll(descendants2);
}
}
// if (children != null && children.size() > 0) {
// descendants.addAll(children);
// for (T t : children) {
// @SuppressWarnings("unchecked")
// List<T> descendants2 = this.getDescendants((PK) t.getId());
// descendants.addAll(descendants2);
// }
// }
return descendants;
}
@Override
public List<T> getAncestors(PK id) {
List<T> ancestors = new ArrayList<T>();
T entity = this.treeDao.getOne(id);
T parent = entity.getParent();
while (parent != null) { // 追述祖先
ancestors.add(parent);
entity = parent;
parent = entity.getParent();
}
return ancestors;
}
}

83
cl_search_api/src/main/java/com/bfd/mf/common/service/cache/SubjectClusterService.java
View File

@ -1,83 +0,0 @@
package com.bfd.mf.common.service.cache;
import com.bfd.mf.common.service.ISubjectClusterService;
import com.bfd.mf.common.service.common.CrudService;
import com.bfd.mf.common.service.es.ClusterService;
import com.bfd.mf.common.util.ESServerUtils;
import com.bfd.mf.common.web.entity.mysql.cache.Cluster;
import com.bfd.mf.common.web.entity.mysql.cache.SubjectCluster;
import com.bfd.mf.common.web.repository.mysql.cache.SubjectClusterRepository;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.client.transport.TransportClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.math.BigInteger;
@Service
public class SubjectClusterService extends CrudService<SubjectCluster, SubjectClusterRepository> implements ISubjectClusterService {
private static final long serialVersionUID = -9149895974179885940L;
private static Logger logger = LoggerFactory.getLogger(SubjectClusterService.class);
@Autowired
private ClusterService clusterService;
@Autowired
private ESServerUtils esServerUtils;
@Autowired
public void setRepo(SubjectClusterRepository repo) {
this.repo = repo;
}
@Override
public SubjectCluster copy(SubjectCluster from, SubjectCluster to) {
to = from;
return to;
}
@Override
public int createIndexBySubject(Long companyId, Long subjectId) {
return 0;
}
/**
* dj-2020-01-10 商情前台 删除 话题检测的 话题
*/
@Override
public boolean deleteIndexBySubject(Long companyId, Long subjectId) {
boolean flag = true;
try {
SubjectCluster subjectCluster = repo.findSubjectClusterBySubjectId(new BigInteger(subjectId.toString()));
if (null == subjectCluster) {
logger.info("[deleteIndexBySubject method] subject id not exit index,the subject is is {}", subjectId);
return true;
}
Cluster cluster = clusterService.get(subjectCluster.getClusterId());
TransportClient client = esServerUtils.getClientByClusterId(cluster.getId());
String indexName = cluster.getPrefixIndexPattern() + "_" + companyId + "_" + subjectId;
IndicesExistsRequest inExistsRequest = new IndicesExistsRequest(indexName);
IndicesExistsResponse inExistsResponse = client.admin().indices()
.exists(inExistsRequest).actionGet();
if (inExistsResponse.isExists()) {
DeleteIndexResponse deleteResponse = client.admin().indices().prepareDelete(indexName).get();
if (deleteResponse.isAcknowledged()) {
if (!client.admin().indices()
.exists(inExistsRequest).actionGet().isExists()) {
logger.info("[deleteIndexBySubject method] the subject cluster info is {} ", subjectCluster);
this.repo.deleteById(subjectCluster.getId());
}
}
}
} catch (Exception e) {
flag = false;
logger.error("[deleteIndexBySubject method] the error is ", e);
}
return flag;
}
}

120
cl_search_api/src/main/java/com/bfd/mf/common/service/cache/TopicQueryService.java
View File

@ -1,120 +0,0 @@
package com.bfd.mf.common.service.cache;
import com.bfd.mf.common.service.es.ESCommonService;
import com.bfd.mf.common.service.es.ParseSearchScopeService;
import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.mf.common.web.vo.params.QueryRequest;
import com.bfd.nlp.common.util.object.TObjectUtils;
import org.elasticsearch.index.query.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.*;
@Service
public class TopicQueryService {
private static Logger logger = LoggerFactory.getLogger(TopicQueryService.class);
private static final String SPLIT = "_";
@Autowired
private ESCommonService esCommonService;
/**
* 根据 condition 中的查询条件组装 查询语句
*/
public BoolQueryBuilder queryByConditions_v1(QueryRequest queryRequest) {
logger.info("[TopicQueryService] queryByConditions_v1 ");
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
// channel
if(null != queryRequest.getSiteTypes() && !("").equals(queryRequest.getSiteTypes())) {
QueryBuilder channelQuery = buildChannelQueryBuilder(queryRequest);
boolQuery.filter(channelQuery);
}
// searchArea
if(("").equals(queryRequest.getSearchArea())){
logger.info("[TopicQueryService] queryByConditions_v1 查询全部地区");
}else{
String searchArea = getSearchArea(queryRequest.getSearchArea());
boolQuery.must(QueryBuilders.termQuery(ESConstant.AREA,searchArea));
}
// cid
if(("").equals(queryRequest.getCid())){
logger.info("[TopicQueryService] queryByConditions_v1 查询全部站点");
}else{
String cid = queryRequest.getCid();
boolQuery.must(QueryBuilders.termQuery(ESConstant.EN_SOURCE,cid.toLowerCase()));
}
//crawlDataFlag
if(("").equals(queryRequest.getCrawlDataFlag())){
logger.info("[TopicQueryService] queryByConditions_v1 查询全部任务");
}else{
String crawlDataFlag = queryRequest.getCrawlDataFlag();
boolQuery.must(QueryBuilders.termQuery(ESConstant.CRAWLDATAFLAG,crawlDataFlag));
}
// pubtime && crawlTime
if(null != queryRequest.getCrawlStartTime() && null != queryRequest.getCrawlEndTime()) {
long crawlStartTime = queryRequest.getCrawlStartTime();
long crawlEndTime = queryRequest.getCrawlEndTime();
boolQuery.must(QueryBuilders.rangeQuery("crawlTime").gte(crawlStartTime).lt(crawlEndTime));
}else{
long startTime = queryRequest.getStartTime();
long endTime = queryRequest.getEndTime();
boolQuery.must(QueryBuilders.rangeQuery("pubTime").gte(startTime).lt(endTime));
}
return boolQuery;
}
private String getSearchArea(String searchArea) {
String area = "中国";
if(searchArea.equals(0)){
area = "美国";
}
if(searchArea.equals(1)){
area = "中国";
}
return area;
}
public QueryBuilder buildChannelQueryBuilder(QueryRequest queryRequest) {
String [] channel = queryRequest.getSiteTypes().split(",");
List<String> channelList =getChannelList(channel);
if (TObjectUtils.isListEmpty(channelList))
return null;
return esCommonService.buildStringQueryByField(ESConstant.CHANNEL, channelList);
}
/**
* 根据 searchScope 中的查询条件组装查询语句
*/
public BoolQueryBuilder buildSearchTextBuilder(Integer searchScope,String keyword) {
return ParseSearchScopeService.getSearchScopeQuery(searchScope,keyword);
}
public BoolQueryBuilder buildSearchTextBuilder(Integer searchScope) {
return ParseSearchScopeService.getSearchScopeQuery(searchScope);
}
public List<String> getChannelList(String [] channel){
List<String> channels = new ArrayList<>();
Map<String,String> map = new HashMap<>();
map.put("0","社交媒体");
map.put("1","网络视频");
map.put("2","网络资讯");
map.put("3","搜索引擎");
map.put("4","电商网站");
map.put("5","垂直网站");
map.put("6","电视广播");
map.put("7","其他网站");
for (String id:channel) {
channels.add(map.get(id));
}
return channels;
}
}

183
cl_search_api/src/main/java/com/bfd/mf/common/service/common/CrudService.java
View File

@ -1,183 +0,0 @@
package com.bfd.mf.common.service.common;
import com.alibaba.fastjson.JSONObject;
import com.bfd.mf.common.web.entity.mysql.Model;
import com.bfd.mf.common.web.vo.params.QueryRequest;
import com.mysql.jdbc.StringUtils;
import org.springframework.data.repository.CrudRepository;
import java.io.Serializable;
import java.util.*;
public abstract class CrudService<M extends Model, R extends CrudRepository<M, Long>> implements Serializable {
public R repo;
public abstract M copy(M from, M to);
public Iterable<M> getAll() {
return this.repo.findAll();
}
public M get(Long id) {
Optional<M> optional = this.repo.findById(id);
return optional.orElse(null);
}
// public abstract void setRepo(R repo);
//
// public M save(M model) {
// return this.repo.save(model);
// }
//
//
//
// public M update(M model) {
// Optional<M> optional = this.repo.findById(model.getId());
// if(optional.isPresent()){
// M updated = copy(model, optional.get());
// return this.repo.save(updated);
// }else {
// return null;
// }
// }
//
// public boolean delete(Long id) {
// this.repo.deleteById(id);
// return true;
// }
//
// public JSONObject buildRootJson(String codeName, int initCapacity, boolean isOrder) {
// JSONObject root = new JSONObject();
// JSONObject codeData = null;
// if (initCapacity > 0) {
// codeData = new JSONObject(initCapacity, isOrder);
// } else {
// codeData = new JSONObject(10, isOrder);
// }
// root.put(codeName, codeData);
// return root;
// }
/*
* 判定一个字符串是否包含另一个字符串中的元素（用于mysql表中判定","号隔开的元素是否存在）
*
* @Param String 已存在的字符串（“,”号隔开）
*
* @Param String 新加入的字符串
*
* @Return boolean 是否已存在
*/
// public boolean strHasExisted(String currentStr, String deltaStr) {
// if (StringUtils.isNullOrEmpty(currentStr) || StringUtils.isNullOrEmpty(deltaStr)) {
// return false;
// }
// currentStr = currentStr.toLowerCase();
// deltaStr = deltaStr.toLowerCase();
//
// String[] nowParams = currentStr.split(",");
// if (null == nowParams || nowParams.length == 0) {
// return false;
// }
// HashSet<String> set = new HashSet<String>();
// set.addAll(Arrays.asList(nowParams));
// //
// String[] deltaParams = deltaStr.split(",");
// for (String delta : deltaParams) {
// if (set.contains(delta)) {
// return true;
// }
// }
// return false;
// }
/*
* 判定一个字符串是否完全包含另一个字符串中的元素
*
* @Param String 已存在的字符串（“,”号隔开）
*
* @Param String 新加入的字符串
*
* @Return boolean 是否全部已存在
*/
// public boolean strTotallyExisted(String currentStr, String deltaStr) {
// if (StringUtils.isNullOrEmpty(currentStr) || StringUtils.isNullOrEmpty(deltaStr)) {
// return false;
// }
// currentStr = currentStr.toLowerCase();
// deltaStr = deltaStr.toLowerCase();
//
// String[] nowParams = currentStr.split(",");
// if (null == nowParams || nowParams.length == 0) {
// return false;
// }
// HashSet<String> set = new HashSet<String>();
// set.addAll(Arrays.asList(nowParams));
// boolean falg = true;
// String[] deltaParams = deltaStr.split(",");
// for (String delta : deltaParams) {
// if (!set.contains(delta)) {
// return false;
// }
// }
// return falg;
// }
/*
* 合并现有字符串和新加入字符串
*
* @Param String 已存在的字符串（“,”号隔开）
*
* @Param String 新加入的字符串
*
* @Return String 合并后的字符串
*/
// public String[] mergeExistedAndDelta(String currentStr, String deltaStr) {
// if (StringUtils.isNullOrEmpty(currentStr) || StringUtils.isNullOrEmpty(deltaStr)) {
// return null;
// }
// currentStr = currentStr.toLowerCase();
// deltaStr = deltaStr.toLowerCase();
//
// String[] nowParams = currentStr.split(",");
// if (null == nowParams || nowParams.length == 0) {
// return deltaStr.split(",");
// }
//
// HashSet<String> set = new HashSet<String>();
// set.addAll(Arrays.asList(nowParams));
// String[] deltaParams = deltaStr.split(",");
// for (String delta : deltaParams) {
// if (!set.contains(delta)) {
// set.add(delta);
// }
// }
// String[] array = new String[set.size()];
// return set.toArray(array);
// }
//
// public String excludeStr(String current, String delta) {
// if (!strHasExisted(current, delta)) {
// return null;
// }
// String[] strs = current.split(",");
// List<String> lst = new ArrayList<String>();
// for (String st : strs) {
// if (!st.equalsIgnoreCase(delta)) {
// lst.add(st);
// }
// }
// StringBuilder sb = new StringBuilder();
// //
// for (int i = 0; i < lst.size(); i++) {
// String s = lst.get(i);
// sb.append(s);
// if (i < lst.size() - 1) {
// sb.append(",");
// }
// }
// return sb.toString();
// }
}

34
cl_search_api/src/main/java/com/bfd/mf/common/service/emotion/EmotionEntityService.java
View File

@ -1,34 +0,0 @@
package com.bfd.mf.common.service.emotion;
import com.bfd.mf.common.service.es.ESCommonService;
import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.nlp.common.util.object.TObjectUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Map;
@Service
public class EmotionEntityService {
private static Logger logger = LoggerFactory.getLogger(EmotionEntityService.class);
@Autowired
private ESCommonService esCommonService;
/**
* 构造情感主题,查询Query
*/
public QueryBuilder buildEmotionEntityBuilder(List<String> emotionList) {
if (TObjectUtils.isListEmpty(emotionList)) {
return null;
}
return esCommonService.buildStringQueryByField(ESConstant.SYS_SENTIMENT, emotionList);
}
}

33
cl_search_api/src/main/java/com/bfd/mf/common/service/es/ClusterService.java
View File

@ -1,33 +0,0 @@
package com.bfd.mf.common.service.es;
import com.bfd.mf.common.service.common.CrudService;
import com.bfd.mf.common.web.entity.mysql.cache.Cluster;
import com.bfd.mf.common.web.repository.mysql.cache.ClusterRepository;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Service
public class ClusterService extends CrudService<Cluster, ClusterRepository> {
private static final long serialVersionUID = -8165362285610331623L;
@Autowired
public void setRepo(ClusterRepository repo) {
this.repo = repo;
}
@Override
public Cluster copy(Cluster from, Cluster to) {
to = from;
return to;
}
/**
* 只对应获取 mini && normal cluster
*/
public Cluster findClusterByType(int type) {
if (type == Cluster.CLUSTER_TYPE.mini_cluster_type || type == Cluster.CLUSTER_TYPE.normal_cluster_type) {
return repo.findClusterByType(type);
}
return null;
}
}

240
cl_search_api/src/main/java/com/bfd/mf/common/service/es/ESCommonService.java
View File

@ -1,240 +0,0 @@
package com.bfd.mf.common.service.es;
import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.mf.common.util.ESServerUtils;
import com.bfd.nlp.common.util.object.TObjectUtils;
import com.bfd.nlp.common.util.string.TStringUtils;
import com.google.common.base.Splitter;
import org.elasticsearch.index.query.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.*;
import static org.elasticsearch.index.query.QueryBuilders.rangeQuery;
@Service
public class ESCommonService {
private final static Logger logger = LoggerFactory.getLogger(ESCommonService.class);
@Autowired
ESServerUtils esServerUtils;
// @Autowired
// FavourService favourService;
public QueryBuilder buildKeyWordsQueryBuilder(String keyWords, String type) {
QueryBuilder keyWordsQuery = null;
if (!TStringUtils.isEmpty(keyWords)) {
if (ESConstant.SEARCH_SCOPE_TYPE_ALL.toLowerCase().equals(type.toLowerCase())) {
/**设置多值查询*/
MatchPhraseQueryBuilder titleQuery = QueryBuilders.matchPhraseQuery(ESConstant.SEARCH_SCOPE_TYPE_TITLE, keyWords).slop(0);
MatchPhraseQueryBuilder contentQuery = QueryBuilders.matchPhraseQuery(ESConstant.SEARCH_SCOPE_TYPE_CONTENT, keyWords).slop(0);
MatchPhraseQueryBuilder forwardContentQuery = QueryBuilders.matchPhraseQuery(ESConstant.SEARCH_SCOPE_TYPE_FORWARD, keyWords).slop(0);
keyWordsQuery = QueryBuilders.boolQuery().should(titleQuery).should(contentQuery).should(forwardContentQuery);
}
if (ESConstant.SEARCH_SCOPE_TYPE_CONTENT.toLowerCase().equals(type.toLowerCase()) || ESConstant.SEARCH_SCOPE_TYPE_TITLE.toLowerCase().equals(type.toLowerCase())) {
keyWordsQuery = QueryBuilders.matchPhraseQuery(type, keyWords).slop(0);
}
if (ESConstant.SEARCH_SCOPE_TYPE_KEYWORDS.toLowerCase().equals(type.toLowerCase())) {
/**热度分析数据展示*/
String wildcardStr = "*" + keyWords + "*";
keyWordsQuery = QueryBuilders.wildcardQuery(ESConstant.HL_KEYWORDS, wildcardStr);
}
if (ESConstant.SOURCE.toLowerCase().equals(type.toLowerCase()) || ESConstant.SEARCH_SCOPE_TYPE_AUTHOR.toLowerCase().equals(type.toLowerCase())) {
/**媒体分析数据展示*/
keyWordsQuery = QueryBuilders.termQuery(type, keyWords);
}
if (ESConstant.WEIBO_AUTHOR.toLowerCase().equals(type.toLowerCase())) {
/**微博用户分析数据展示
* 1:针对的是微博的数据；所以把查询条件限制在weibo
* @FIXED：数据监控按照来源进行搜索；会把多余的信息搜索出来
**/
keyWordsQuery = QueryBuilders.boolQuery().
should(QueryBuilders.boolQuery().filter(QueryBuilders.wildcardQuery(type, "*" + keyWords + "*")).filter(QueryBuilders.termQuery(ESConstant.DOC_TYPE, ESConstant.WEI_BO)));
//之前的“作者”字段，不再能搜出来微信公众号，只能按作者字段值搜索(V1.1.2版本需求说明)
//should(QueryBuilders.boolQuery().filter(QueryBuilders.wildcardQuery(ESConstant.SOURCE, "*" + keyWords + "*")).filter(QueryBuilders.termQuery(ESConstant.DOC_TYPE, ESConstant.WEI_XIN)))
}
if (ESConstant.WEIBO_USER_SEX.toLowerCase().equals(type.toLowerCase())) {
/**微博用户男女数据展示
*
* 1:female
* 2:male
* 3:other/""
*/
if (ESConstant.WEIBO_USER_SEX_FEMALE.toLowerCase().equals(keyWords.toLowerCase()) || ESConstant.WEIBO_USER_SEX_MALE.toLowerCase().equals(keyWords.toLowerCase())) {
keyWordsQuery = QueryBuilders.termQuery(type, keyWords);
} else {
keyWordsQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(type, Arrays.asList(new String[]{ESConstant.WEIBO_USER_SEX_FEMALE, ESConstant.WEIBO_USER_SEX_MALE})));
}
}
if (ESConstant.WEIBO_USER_VERIFIED.toLowerCase().equals(type.toLowerCase())) {
List<String> verifiedList = Splitter.on(",").trimResults().splitToList(keyWords);
StringBuilder stringBuilder = buildBuilder(null, verifiedList);
// weibo 个人认证
keyWordsQuery = QueryBuilders.simpleQueryStringQuery(stringBuilder.toString())
.field(ESConstant.WEIBO_USER_VERIFIED).defaultOperator(Operator.OR);
}
if (ESConstant.WEIBO_USER_PROVINCE.toLowerCase().equals(type.toLowerCase())) {
/**province*/
String wildcardStr = keyWords + "*";
keyWordsQuery = QueryBuilders.wildcardQuery(ESConstant.WEIBO_USER_PROVINCE, wildcardStr);
}
if (ESConstant.URL.toLowerCase().equals(type.toLowerCase())) {
/**依据URL查询*/
keyWordsQuery = QueryBuilders.termQuery(type, keyWords);
}
if (ESConstant._ALL.toLowerCase().equals(type.toLowerCase())) {
/**依据_all查询*/
keyWordsQuery = QueryBuilders.queryStringQuery(keyWords).defaultField(ESConstant._ALL);
}
if (ESConstant.DOC_ID.toLowerCase().equals(type.toLowerCase())) {
/**依据_all查询*/
keyWordsQuery = QueryBuilders.termQuery(type, keyWords);
}
if (ESConstant.TITLE_SIMHASH.toLowerCase().equals(type.toLowerCase())) {
/**依据_all查询*/
keyWordsQuery = QueryBuilders.termQuery(type, keyWords);
}
if (ESConstant.DOC_TYPE.toLowerCase().equals(type.toLowerCase())) {
keyWordsQuery = QueryBuilders.termQuery(type, keyWords);
}
//微信公众号
if (ESConstant.SEARCH_SCOPE_WECHAT_SUBSCRIPTION.toLowerCase().equals(type.toLowerCase())) {
keyWordsQuery = QueryBuilders.wildcardQuery(ESConstant.SOURCE, "*" + keyWords + "*");
}
}
return keyWordsQuery;
}
public QueryBuilder buildRangeQueryBuilder(String field, Object startVal, Object endVal, Boolean isIncludeLower, Boolean isIncludeUpper) {
return rangeQuery(field)
.from(startVal)
.to(endVal)
.includeLower(isIncludeLower)
.includeUpper(isIncludeUpper);
}
public QueryBuilder buildStringQueryByField(String field, List<String> lists) {
if (null == field || null == lists || lists.size() == 0) {
return QueryBuilders.simpleQueryStringQuery(ESConstant.OPERATOR_ALL);
}
int size = lists.size();
if (ESConstant.DOC_TYPE.equals(field)) {
if (ESConstant.TYPE_LIST.size() == size) {
return QueryBuilders.simpleQueryStringQuery(ESConstant.OPERATOR_SINGLE_ALL).field(ESConstant.DOC_TYPE).defaultOperator(Operator.OR);
} else {
StringBuilder strBuilder = buildBuilder(field, lists);
return QueryBuilders.simpleQueryStringQuery(
strBuilder.toString())
.field(ESConstant.DOC_TYPE)
.defaultOperator(Operator.OR);
}
} else if (ESConstant.SYS_SENTIMENT.equals(field)
|| (ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_VALUE).equals(field)
|| (ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_VALUE).equals(field)) {
BoolQueryBuilder emotionQuery = QueryBuilders.boolQuery();
for (String emotion : lists) {
switch (emotion) {
case ESConstant.COMMON_TAG:
emotionQuery.should(buildRangeQueryBuilder(field, ESConstant.SENTIMENTAL_MID, ESConstant.SENTIMENTAL_MAX, true, true));
//emotionQuery.should(QueryBuilders.matchPhraseQuery(ESConstant.SYS_SENTIMENT, ESConstant.COMMON_TAG_VALUE).slop(0).boost(1));
break;
case ESConstant.NEGATIVE_TAG:
emotionQuery.should(buildRangeQueryBuilder(field, ESConstant.SENTIMENTAL_MIN, ESConstant.SENTIMENTAL_THRESHOLD, true, true));
//emotionQuery.should(QueryBuilders.matchPhraseQuery(ESConstant.SYS_SENTIMENT, ESConstant.NEGATIVE_TAG_VALUE).slop(0).boost(1));
break;
case ESConstant.NEUTER_TAG:
emotionQuery.should(buildRangeQueryBuilder(field, ESConstant.SENTIMENTAL_THRESHOLD, ESConstant.SENTIMENTAL_MID, false, false));
//emotionQuery.should(QueryBuilders.matchPhraseQuery(ESConstant.SYS_SENTIMENT, ESConstant.NEUTER_TAG_VALUE).slop(0).boost(1));
break;
default:
break;
}
}
return emotionQuery;
} else if (ESConstant.TITLE_SIMHASH.equals(field)) {
return QueryBuilders.termsQuery(ESConstant.TITLE_SIMHASH, lists);
} else if (ESConstant.DOC_ID.equals(field)) {
return QueryBuilders.termsQuery(ESConstant.DOC_ID, lists);
} else if (ESConstant.DATA_ID.equals(field)) {
return QueryBuilders.termsQuery(ESConstant.DATA_ID, lists);
} else if (ESConstant.CONTENT_TAG.equals(field)) {
return TObjectUtils.isListEmpty(lists) ? null : QueryBuilders.simpleQueryStringQuery(buildContentTagOrOtherQueryString(lists)).field(field).defaultOperator(Operator.OR);
} else {
StringBuilder strBuilder = buildBuilder(field, lists);
return QueryBuilders.simpleQueryStringQuery(strBuilder.toString()).field(field).defaultOperator(Operator.OR);
}
}
public StringBuilder buildBuilder(String field, List<String> lists) {
int size = lists.size();
StringBuilder strBuilder = new StringBuilder();
for (int i = 0; i < size; i++) {
String str = lists.get(i);
if (i == (size - 1)) {
strBuilder.append(str);
} else {
if (ESConstant.ZIXUN.equals(str)) {
strBuilder.append(ESConstant.NEWS).append(ESConstant.OPERATOR_SPACE);
strBuilder.append(ESConstant.APP).append(ESConstant.OPERATOR_SPACE);
}
strBuilder.append(str).append(ESConstant.OPERATOR_SPACE);
}
}
return strBuilder;
}
private String buildContentTagOrOtherQueryString(List<String> lists) {
StringBuilder strBuilder = new StringBuilder();
for (int i = 0; i < lists.size(); i++) {
String str = lists.get(i);
if (i == (lists.size() - 1)) {
strBuilder.append(str);
} else {
strBuilder.append(str).append(ESConstant.OPERATOR_SPACE);
}
}
return strBuilder.toString();
}
// public List<Integer> miniAndPrimaryClusterType() {
// List<Integer> typeList = new ArrayList<>();
// typeList.add(Cluster.CLUSTER_TYPE.mini_cluster_type);
// typeList.add(Cluster.CLUSTER_TYPE.normal_cluster_type);
// return typeList;
// }
// public QueryBuilder buildFavourQueryBuilder(Long bookMarkId, AbsRequest absRequest) {
// QueryBuilder bookMarkQuery = null;
// List<String> docIdLists = new ArrayList<>();
// List<FavourDoc> favourList = null;
//// if (null == bookMarkId) {
//// bookMarkQuery = null;
//// } else if (bookMarkId == -1) {
//// favourList = favourService.getAllFavDoc(absRequest);
////
//// } else {
//// List<Long> lists = new ArrayList<>();
//// lists.add(bookMarkId);
//// favourList = favourService.getFavourDocList(absRequest, lists);
//// }
// if (null != favourList && favourList.size() > 0) {
// for (FavourDoc favourDoc : favourList) {
// docIdLists.add(favourDoc.getDocId());
// }
// bookMarkQuery = buildStringQueryByField(ESConstant.DOC_ID, docIdLists);
// }
// return bookMarkQuery;
// }
}

82
cl_search_api/src/main/java/com/bfd/mf/common/service/es/MonitorMainSourceService.java
View File

@ -1,82 +0,0 @@
package com.bfd.mf.common.service.es;
import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.mf.common.web.entity.mysql.report.MainSourceEntity;
import com.bfd.mf.common.web.repository.mysql.UserRepository;
import com.bfd.mf.common.web.repository.mysql.report.MainSourceRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.*;
@Service
public class MonitorMainSourceService {
private static Logger logger = LoggerFactory.getLogger(MonitorMainSourceService.class);
@Autowired
private MainSourceRepository mainSourceRepository;
@Autowired
private UserRepository userRepository;
public Map<String, Set<String>> getMainSourceMap(String token) {
Map<String, Set<String>> mainSourceMap = new HashMap<>();
Long companyId = userRepository.findByTokenAndDel(token, false).getCompanyId();
if (companyId == null) {
return mainSourceMap;
}
List<MainSourceEntity> mainSourceList = mainSourceRepository.findByCompanyIdAndStatus(companyId, 1);
for (MainSourceEntity mainSourceEntity : mainSourceList) {
String channel = mainSourceEntity.getDocTypeEn();
Set<String> sourceOfChannel;
if (mainSourceMap.containsKey(channel)) {
sourceOfChannel = mainSourceMap.get(channel);
} else {
sourceOfChannel = new HashSet<>();
}
sourceOfChannel.add(mainSourceEntity.getSource());
mainSourceMap.put(channel, sourceOfChannel);
}
return mainSourceMap;
}
public Float getAlexaScore(String docType, Map<String, Set<String>> mainSourceMap, String source, Float alexaTmpScore) {
Float alexaScore = ESConstant.ALEXA_DEFAULT_SCORE;
if (null != docType && (ESConstant.NEWS.equals(docType) || ESConstant.MEDIA.equals(docType) || ESConstant
.WEI_XIN.equals(docType))) {
alexaScore = (null == alexaTmpScore) ? alexaScore : alexaTmpScore;
}
// 用户设置的主来源列表为首位
if (mainSourceMap.containsKey(docType) && mainSourceMap.get(docType).contains(source)) {
alexaScore = (1 - 1.0f / (1 + alexaScore));
}
return alexaScore;
}
// public void sortByMainScoreBase(List<ESMonitorBaseEntity> timeSeries, String sortFlag, DataQueryRequest dataQueryRequest) {
// try {
// Map<String ,Float> siteMap = AlexaSingleUtils.getInstance(sourceAlexaManager).getAlexaScoreBySiteName();
//// Map<String, Set<String>> mainSourceMap = getMainSourceMap(dataQueryRequest.getToken());
// if (sortFlag.equals(ESConstant.SCORE_DESC_SORT_FLAG)) {
// for(ESMonitorBaseEntity esMonitorBaseEntity : timeSeries){
// Float sore = esMonitorBaseEntity.getScore();
//// Float alexaScore = getAlexaScore(esMonitorBaseEntity.getDocType(),mainSourceMap,esMonitorBaseEntity.getSource(),siteMap.get
//// (esMonitorBaseEntity.getSource()));
// Float emotionScore = esMonitorBaseEntity.getSysSentimentTag().equals(ESConstant.NEGATIVE_TAG) ? 0.5f * esMonitorBaseEntity.getSimilarityNumber() : 0f;
// Float docNumAndEmotionSore = esMonitorBaseEntity.getSimilarityNumber() + emotionScore;
//// Float resultScore = 5 * sore / (1 + sore) + 3 * (1 - alexaScore / (1 + alexaScore)) + docNumAndEmotionSore / (1 + docNumAndEmotionSore);
//// esMonitorBaseEntity.setScore(resultScore);
// }
// }
// Collections.sort(timeSeries);
// } catch (Exception e) {
// logger.error("[sortByMainScore] 重要度排序出现异常:{}", e);
// }
// }
}

92
cl_search_api/src/main/java/com/bfd/mf/common/service/es/ParseSearchScopeService.java
View File

@ -1,92 +0,0 @@
package com.bfd.mf.common.service.es;
import com.bfd.mf.common.web.vo.params.QueryRequest;
import com.bfd.nlp.common.util.string.TStringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.springframework.beans.factory.annotation.Autowired;
import java.util.List;
public class ParseSearchScopeService {
/**
* 二次检索 条件组装
*/
public static BoolQueryBuilder getSearchScopeQuery(Integer searchScope,String keyword) {
BoolQueryBuilder boolQueryBuilder = null;
QueryBuilder queryBuilder = null;
QueryRequest queryRequest = new QueryRequest();
List<String> searchList = queryRequest.getSearchScopeValue(searchScope);
System.out.println(searchList.toString());
if (!TStringUtils.isEmpty(keyword)) {
boolQueryBuilder = QueryBuilders.boolQuery().minimumShouldMatch(1);
if(searchList.size() ==1){
queryBuilder = QueryBuilders.matchPhraseQuery(searchList.get(0), keyword).slop(0);
}else{
if(searchList.toString().contains("content") && searchList.toString().contains("title") ){
MatchPhraseQueryBuilder titleQuery = QueryBuilders.matchPhraseQuery("title", keyword).slop(0);
MatchPhraseQueryBuilder contentQuery = QueryBuilders.matchPhraseQuery("content", keyword).slop(0);
queryBuilder = QueryBuilders.boolQuery().should(titleQuery).should(contentQuery);
}
if(searchList.toString().contains("content") && searchList.toString().contains("author")){
MatchPhraseQueryBuilder authorQuery = QueryBuilders.matchPhraseQuery("author", keyword).slop(0);
MatchPhraseQueryBuilder contentQuery = QueryBuilders.matchPhraseQuery("content", keyword).slop(0);
queryBuilder = QueryBuilders.boolQuery().should(authorQuery).should(contentQuery);
}
if(searchList.toString().contains("title") && searchList.toString().contains("author")){
MatchPhraseQueryBuilder authorQuery = QueryBuilders.matchPhraseQuery("author", keyword).slop(0);
MatchPhraseQueryBuilder titleQuery = QueryBuilders.matchPhraseQuery("title", keyword).slop(0);
queryBuilder = QueryBuilders.boolQuery().should(authorQuery).should(titleQuery);
}
}
boolQueryBuilder.should(queryBuilder);
}
return boolQueryBuilder;
}
// if (!TStringUtils.isEmpty(keyword)) { // 如果查询的关键词为空的话，就表示不做二次查询
// boolQueryBuilder = QueryBuilders.boolQuery().minimumShouldMatch(1);
// if(searchScope == 0){ // 0 content 1 title 2 author 3 con+tit 4con+aut 5con+com
// queryBuilder = QueryBuilders.matchPhraseQuery("content", keyword).slop(0);
// }
// if(searchScope == 1){
// queryBuilder = QueryBuilders.matchPhraseQuery("title", keyword).slop(0);
// }
// if(searchScope == 2){
// queryBuilder = QueryBuilders.matchPhraseQuery("author", keyword).slop(0);
// }
// if(searchScope == 3){
// MatchPhraseQueryBuilder titleQuery = QueryBuilders.matchPhraseQuery("title", keyword).slop(0);
// MatchPhraseQueryBuilder contentQuery = QueryBuilders.matchPhraseQuery("content", keyword).slop(0);
// queryBuilder = QueryBuilders.boolQuery().should(titleQuery).should(contentQuery);
// }
// if(searchScope == 4){
// MatchPhraseQueryBuilder titleQuery = QueryBuilders.matchPhraseQuery("author", keyword).slop(0);
// MatchPhraseQueryBuilder contentQuery = QueryBuilders.matchPhraseQuery("content", keyword).slop(0);
// queryBuilder = QueryBuilders.boolQuery().should(titleQuery).should(contentQuery);
// }
// if(searchScope == 5){
// MatchPhraseQueryBuilder titleQuery = QueryBuilders.matchPhraseQuery("title", keyword).slop(0);
// MatchPhraseQueryBuilder contentQuery = QueryBuilders.matchPhraseQuery("content", keyword).slop(0);
// queryBuilder = QueryBuilders.termQuery("primary","0");
// queryBuilder = QueryBuilders.boolQuery().should(titleQuery).should(contentQuery);
// }
// }
public static BoolQueryBuilder getSearchScopeQuery(Integer searchScope) {
BoolQueryBuilder searchScopeQuery = null;
System.out.println("得先到这个方法吧？？？？？");
if(searchScope < 4){
searchScopeQuery = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("primary", 0));
}else {
searchScopeQuery = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("primary", 1));
}
return searchScopeQuery;
}
}

266
cl_search_api/src/main/java/com/bfd/mf/common/service/text/TextService.java
View File

@ -1,266 +0,0 @@
package com.bfd.mf.common.service.text;
import com.alibaba.fastjson.JSONObject;
import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.mf.common.web.vo.view.monitor.ESMonitorEntity;
import com.bfd.mf.config.BFDApiConfig;
import net.logstash.logback.encoder.org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HeaderElement;
import org.apache.http.HeaderElementIterator;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.conn.ConnectionKeepAliveStrategy;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicHeaderElementIterator;
import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
public class TextService {
private static Logger logger = LoggerFactory.getLogger(TextService.class);
@Autowired
private BFDApiConfig bfdApiConfig;
public Map<String, Integer> CloudPost(Map<String, Integer> map, Integer type) {
CloseableHttpClient httpclient = getClientFromPool();
HttpPost httpPost = new HttpPost(bfdApiConfig.getTextPostUrl());
String result = null;
try {
JSONObject params = new JSONObject();
params.put("map", JSONObject.toJSONString(map));
params.put("type", type);
StringEntity strEntity = new StringEntity(JSONObject.toJSONString(map), "utf-8");
strEntity.setContentType("application/json");
httpPost.setEntity(strEntity);
long start = System.currentTimeMillis();
HttpResponse response = httpclient.execute(httpPost);
System.err.println(Thread.currentThread().getName() + " single data used Time: " + (System.currentTimeMillis() - start));
if (response != null) {
HttpEntity resEntity = response.getEntity();
if (resEntity != null) {
result = EntityUtils.toString(resEntity, "utf-8");
EntityUtils.consume(resEntity);
map = (Map<String, Integer>) JSONObject.parse(result);
}
}
} catch (IOException e) {
logger.error("Http Post For Spam Tag : " + e);
}
return map;
}
public List<JSONObject> emotionPost(List<JSONObject> list) throws Exception {
CloseableHttpClient httpclient = getClientFromPool();
HttpPost httpPost = new HttpPost(bfdApiConfig.getEmotionPostUrl());
String result;
try {
StringEntity strEntity = new StringEntity(JSONObject.toJSONString(list), "utf-8");
// System.out.println(JSONObject.toJSONString(list));
strEntity.setContentType("application/json");
httpPost.setEntity(strEntity);
long start = System.currentTimeMillis();
HttpResponse response = httpclient.execute(httpPost);
System.err.println(Thread.currentThread().getName() + " single data used Time: " + (System.currentTimeMillis() - start));
if (response != null) {
HttpEntity resEntity = response.getEntity();
if (resEntity != null) {
result = EntityUtils.toString(resEntity, "utf-8");
EntityUtils.consume(resEntity);
list = (List<JSONObject>) JSONObject.parse(result);
}
}
} catch (IOException e) {
logger.error("Http Post For Spam Tag : " + e);
throw new Exception("Http Post For Spam"+ExceptionUtils.getFullStackTrace(e));
}
return list;
}
public Map<String, Integer> post(Map<String, Integer> map, Integer type) {
CloseableHttpClient httpclient = getClientFromPool();
String url = "";
if (type == 2) {
url = bfdApiConfig.getTextPostUrl();
} else {
url = bfdApiConfig.getWordCloudPostUrl();
}
System.out.println("???"+ url);
HttpPost httpPost = new HttpPost(url);
String result = null;
try {
StringEntity strEntity = new StringEntity(JSONObject.toJSONString(map), "utf-8");
strEntity.setContentType("application/json");
httpPost.setEntity(strEntity);
long start = System.currentTimeMillis();
HttpResponse response = httpclient.execute(httpPost);
System.err.println(Thread.currentThread().getName() + " single data used Time: " + (System.currentTimeMillis() - start));
if (response != null) {
HttpEntity resEntity = response.getEntity();
if (resEntity != null) {
result = EntityUtils.toString(resEntity, "utf-8");
EntityUtils.consume(resEntity);
map = (Map<String, Integer>) JSONObject.parse(result);
map = replace(map);
}
}
} catch (IOException e) {
logger.error("Http Post For Spam Tag : " + e);
}
return map;
}
private Map<String, Integer> replace(Map<String, Integer> map) {
Map<String, Integer> resultMap = new HashMap<String, Integer>();
String E1 = "[`~!@$%^&*()+=|{}':;',.<>/?~！@￥%……&*（）——+|{}【】‘；：”“’。，、？_]";
String E2 = "[a-zA-Z]";// 英文
String E3 = "[0-9]";// 数字
String values = "";
for (Map.Entry<String, Integer> entry : map.entrySet()) {
values = entry.getKey().replaceAll(E1, "");
values = values.replace(E2, "");
values = values.replace(E3, "");
if(StringUtils.isNotEmpty(values)) {
resultMap.put(values, entry.getValue());
}
}
return resultMap;
}
public CloseableHttpClient getClientFromPool() {
return HttpClients.custom()
.setConnectionManager(new PoolingHttpClientConnectionManager())
.setKeepAliveStrategy(defaultStrategy())
.setDefaultRequestConfig(RequestConfig.custom().setStaleConnectionCheckEnabled(true).build())
.build();
}
private ConnectionKeepAliveStrategy defaultStrategy() {
return new ConnectionKeepAliveStrategy() {
@Override
public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
HeaderElementIterator it = new BasicHeaderElementIterator
(response.headerIterator(HTTP.CONN_KEEP_ALIVE));
while (it.hasNext()) {
HeaderElement he = it.nextElement();
String param = he.getName();
String value = he.getValue();
if (value != null && param.equalsIgnoreCase
("timeout")) {
return Long.parseLong(value) * 1000;
}
}
return 60 * 1000;//如果没有约定，则默认定义时长为60s
}
};
}
/**
* 获取最新情感值
*/
public void resetDataEmotion(List<ESMonitorEntity> monitorList) {
if (monitorList != null && monitorList.size() > 0) {
List<JSONObject> objectList = new ArrayList<>();
JSONObject jsonObject;
for (ESMonitorEntity esMonitorEntity : monitorList) {
jsonObject = new JSONObject();
jsonObject.put(ESConstant.TITLE, esMonitorEntity.getTitle());
jsonObject.put(ESConstant.SUMMARY, esMonitorEntity.getContentAbstract());
jsonObject.put(ESConstant.SENTIMENT, resetSentiment(esMonitorEntity.getSysSentimentTag()));
objectList.add(jsonObject);
}
try {
objectList = emotionPost(objectList);
} catch (Exception e) {
e.printStackTrace();
}
resetDataEmotion(monitorList, objectList);
}
}
/**
* 重置情感值
*/
public void resetDataEmotion(List<ESMonitorEntity> monitorList, List<JSONObject> jsonObjectList) {
JSONObject jsonObject;
ESMonitorEntity monitorEntity;
for (int i = 0; i < monitorList.size(); i++) {
jsonObject = jsonObjectList.get(i);
monitorEntity = monitorList.get(i);
logger.error("docId sentiment source :{}", monitorEntity.getSysSentimentTag());
monitorEntity.setSysSentimentTag(resetSentiment(jsonObject.getInteger(ESConstant.SENTIMENT)));
logger.error("docId sentiment target {}", monitorEntity.getSysSentimentTag());
}
}
/**
* 情感值与数字之间的转化
*/
public Integer resetSentiment(String sentiment) {
Integer values = 1;
if (ESConstant.NEGATIVE_TAG.equals(sentiment)) {
values = 3;
}
if (ESConstant.COMMON_TAG.equals(sentiment)) {
values = 1;
}
if (ESConstant.NEUTER_TAG.equals(sentiment)) {
values = 2;
}
return values;
}
/**
* 数字与情感值的转化
*/
private String resetSentiment(Integer sentiment) {
String values = ESConstant.COMMON_TAG;
if (sentiment == 1) {
values = ESConstant.COMMON_TAG;
}
if (sentiment == 2) {
values = ESConstant.NEUTER_TAG;
}
if (sentiment == 3) {
values = ESConstant.NEGATIVE_TAG;
}
return values;
}
/**
* 计算每个关键词出现的文章数
*/
private void getKeyWordNewsCount(List<String> list, Map<String, Integer> map) {
for (String s : list) {
for (Map.Entry<String, Integer> entry : map.entrySet()) {
if (map.containsKey(s)) {
map.put(entry.getKey(), entry.getValue() + 1);
}
}
}
}
private void initKeyWordMap(Map<String, Integer> map) {
for (Map.Entry<String, Integer> entry : map.entrySet()) {
map.put(entry.getKey(), 0);
}
}
}

193
cl_search_api/src/main/java/com/bfd/mf/common/thread/SubjectQueryDataService.java
View File

@ -1,193 +0,0 @@
package com.bfd.mf.common.thread;
import com.bfd.mf.common.service.cache.TopicQueryService;
import com.bfd.mf.common.service.es.ESCommonService;
import com.bfd.mf.common.util.ESServerUtils;
import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.mf.common.util.thread.SubjectDataQueryThread;
import com.bfd.mf.common.web.entity.mysql.cache.Cluster;
import com.bfd.mf.common.web.vo.params.QueryRequest;
import com.bfd.nlp.common.util.string.TStringUtils;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchType;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
/**
* 依据时间多线程(请求ES的时间区间范围不可以太长)
*/
@Service
public class SubjectQueryDataService {
private Logger logger = LoggerFactory.getLogger(SubjectQueryDataService.class);
private static final Long ONE_DAY = 60 * 60 * 1000L * 24;
private static final Long clusterId = 3L;
@Autowired
private TopicQueryService topicQueryService;
@Autowired
private ESServerUtils esServerUtils;
@Autowired
private ESCommonService esCommonService;
/**
* 专题下数据列表查询
*/
public List<SearchResponse> fetchResponseDataFromCache(QueryRequest queryRequest,Long clusterId ,
List<String> currentIndexList,
String[] fieldList) {
logger.info("[SubjectQueryDataService ] fetchResponseDataFromCache : 数据列表查询 ");
// 存储结果x
List<SearchResponse> searchResponseList = Collections.synchronizedList(new ArrayList<>());
// 可以分配线程数量
int availableProcessors = Runtime.getRuntime().availableProcessors();
ExecutorService executorService = Executors.newFixedThreadPool(availableProcessors);
TransportClient transportClient = esServerUtils.getClientByClusterId(clusterId);
final CountDownLatch latch = new CountDownLatch(1);
BoolQueryBuilder boolQueryBuilder = topicQueryService.queryByConditions_v1(queryRequest);
// 如果要根据ID 查询数据
if(null != queryRequest.getDocIds() && !("").equals(queryRequest.getDocIds())) {
String docIds = queryRequest.getDocIds();
List<String> dataIdList = new ArrayList<>();
if(docIds.contains(",")){
String ids [] = docIds.split(",");
for (String id:ids) {
dataIdList.add(id);
}
}else{
dataIdList.add(docIds);
}
QueryBuilder queryBuilder = esCommonService.buildStringQueryByField(ESConstant.DOC_ID, dataIdList);
boolQueryBuilder = QueryBuilders.boolQuery().filter(queryBuilder);
}
// 关键词不为空就添加关键词查询语句 = 0 content 1 title 2 author 3 con+tit 4con+aut 5con+com
Integer searchScope = queryRequest.getSearchScope();
String keyword = queryRequest.getKeyword();
BoolQueryBuilder searchTextBuilder = topicQueryService.buildSearchTextBuilder(searchScope);
boolQueryBuilder.filter(searchTextBuilder);
if (TStringUtils.isNotEmpty(keyword)) {
searchTextBuilder = topicQueryService.buildSearchTextBuilder(searchScope,keyword);
}
boolQueryBuilder.filter(searchTextBuilder);
// 组装完整查询语句
SearchRequestBuilder searchCountQuery = esServerUtils
.buildSearchRequestBuilder(clusterId, currentIndexList)
.setQuery(boolQueryBuilder)
.setFetchSource(fieldList, null)
.setSearchType(SearchType.DEFAULT);
System.out.println(searchCountQuery);
executorService.submit(new SubjectDataQueryThread(
searchResponseList,
searchCountQuery,
transportClient,
latch));
try {
latch.await(20L, TimeUnit.SECONDS);
executorService.shutdown();
} catch (InterruptedException e) {
e.printStackTrace();
}
return searchResponseList;
}
/**
根据时间 组装获取 索引列表 时间索引ES 全部数据
*/
public List<String> getIndexListByTimeRange(Cluster cluster, Long startTime,Long endTime) {
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
Set<String> set = new HashSet<>();
for (Long time = startTime; time <= endTime; time += (20 * 60 * 60 * 1000L)) {
set.add(cluster.getPrefixIndexPattern() + "_" + format.format(time));
}
set.add(cluster.getPrefixIndexPattern() + "_" + format.format(endTime));
return new ArrayList<>(set);
}
/**
* 根据 subjectId 列表，组装获取专题索引 专题索引ES 专题数据
*/
public List<String> getIndexBySubjectIds(Cluster cluster, String subjectIds) {
Set<String> set = new HashSet<>();
if(subjectIds.contains(",")){
for(String subjectId :subjectIds.split(",")){
set.add(cluster.getPrefixIndexPattern() + "_"+subjectId);
}
}else {
String subjectId = subjectIds;
set.add(cluster.getPrefixIndexPattern() + "_"+subjectId);
}
return new ArrayList<>(set);
}
//
// public List<SearchResponse> fetchResponseCommentsFromCache(CommentQueryRequest commentQueryRequest,
// List<String> currentIndexList,
// String[] fieldList) {
// logger.info("[ SubjectQueryDataService ] fetchResponseCommentsFromCache : 某个详情的评论列表查询 ");
//
// // 存储结果x
// List<SearchResponse> searchResponseList = Collections.synchronizedList(new ArrayList<>());
// // 可以分配线程数量
// int availableProcessors = Runtime.getRuntime().availableProcessors();
// ExecutorService executorService = Executors.newFixedThreadPool(availableProcessors);
// long clusterId = 3L;
// TransportClient transportClient = esServerUtils.getClientByClusterId(clusterId);
// final CountDownLatch latch = new CountDownLatch(1);
//
// BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
// boolQueryBuilder.must(QueryBuilders.termQuery(ESConstant.DOC_ID,commentQueryRequest.getDocId()));
// boolQueryBuilder.must(QueryBuilders.termQuery(ESConstant.PRIMARY,0));
//
//
// // 组装完整查询语句
// SearchRequestBuilder searchCountQuery = esServerUtils
// .buildSearchRequestBuilder(clusterId, currentIndexList)
// .setQuery(boolQueryBuilder)
// .setFetchSource(fieldList, null)
// .setSearchType(SearchType.DEFAULT);
//
// System.out.println(searchCountQuery);
// executorService.submit(new SubjectDataQueryThread(
// searchResponseList,
// searchCountQuery,
// transportClient,
// latch));
//
// try {
// latch.await(20L, TimeUnit.SECONDS);
// executorService.shutdown();
// } catch (InterruptedException e) {
// e.printStackTrace();
// }
// return searchResponseList;
// }
}

1059
cl_search_api/src/main/java/com/bfd/mf/common/util/ESConstant.java
File diff suppressed because it is too large
View File

118
cl_search_api/src/main/java/com/bfd/mf/common/util/ESServerUtils.java
View File

@ -1,118 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.util;
import com.bfd.mf.common.service.es.ClusterService;
import com.bfd.mf.common.web.entity.mysql.cache.Cluster;
import com.bfd.nlp.common.util.object.TObjectUtils;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.support.IndicesOptions;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.TransportAddress;
import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.io.Serializable;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* 区别多环境问题
*/
@Component
public class ESServerUtils implements Serializable {
private static Logger logger = LoggerFactory.getLogger(ESServerUtils.class);
private static final long serialVersionUID = 1L;
private static Map<Long, TransportClient> clientByClusterId = new HashMap<>();
@Autowired
private ClusterService clusterService;
@PostConstruct
public void initEsServer() {
Iterable<Cluster> iterator = clusterService.getAll();
long startTime = System.currentTimeMillis();
iterator.forEach((final Cluster cluster) ->
{
logger.info("[Initial Elastic Search] Initial ES handler, the cluster name is {}", cluster.getClusterName());
System.setProperty("es.set.netty.runtime.available.processors", "false");
Settings settings = Settings.builder().put("cluster.name",
cluster.getClusterName())
.put("client.transport.sniff", false)
.build();
try {
logger.info("[Initial Elastic Search] Initial First Url= {} and First Port={} clusterName = {}",
cluster.getIp(), cluster.getTcpPort(), cluster.getClusterName());
TransportClient client = new PreBuiltTransportClient(settings);
String[] ports = cluster.getTcpPort().split(",");
int index = 0;
for (String ip : cluster.getIp().split(",")) {
int port = Integer.parseInt(ports[index]);
client.addTransportAddress(new TransportAddress(InetAddress.getByName(ip), port));
index++;
}
logger.info("[Initial Elastic Search] Initial ES Finished.the client is{} ", client);
clientByClusterId.put(cluster.getId(), client);
} catch (UnknownHostException e) {
logger.error("UnknownHostException occurs while creating TransportClient,error message={}", e);
System.exit(1);
} catch (Exception ex) {
logger.error("Exception occurs while creating TransportClient,error message={}", ex);
System.exit(1);
}
logger.info("[Initial Elastic Search] Initial ES handler finish...");
}
);
long endTime = System.currentTimeMillis();
logger.info("[Initial Elastic Search] finish,the used time is {}", (endTime - startTime));
}
public TransportClient getClientByClusterId(Long clusterId) {
return clientByClusterId.get(clusterId);
}
/**
* 根据索引 List 查询指定ES
*/
public SearchRequestBuilder buildSearchRequestBuilder(Long clusterId, List<String> indexList) {
SearchRequestBuilder indexSearch = null;
try {
if (TObjectUtils.isListEmpty(indexList)) {
indexSearch = getClientByClusterId(clusterId).prepareSearch(clusterService.get(clusterId).getPrefixIndexPattern() + "_*");
} else {
indexSearch = getClientByClusterId(clusterId).prepareSearch((String[]) indexList.toArray(new String[indexList.size()]));
}
// allowNoIndices: true,不存在的索引不会抛异常;false,抛异常
indexSearch.setIndicesOptions(IndicesOptions.fromOptions(true, true,
true, false));
} catch (Exception e) {
logger.error("[ESServerUtils] buildSearchRequestBuilder: build SearchRequestBuilder error,the error message is:{}", e);
}
return indexSearch;
}
}

20
cl_search_api/src/main/java/com/bfd/mf/common/util/EsQueryConditionUtils.java
View File

@ -1,20 +0,0 @@
package com.bfd.mf.common.util;
public class EsQueryConditionUtils {
public static String[] getIncludeQueryField(String... includeField) {
String[] includeFields = new String[includeField.length];
System.arraycopy(includeField, 0, includeFields, 0, includeField.length - 1 + 1);
return includeFields;
}
public static String[] getExcludeQueryField(String... excludeField) {
String[] excludeFields = new String[excludeField.length];
System.arraycopy(excludeField, 0, excludeFields, 0, excludeField.length - 1 + 1);
return excludeFields;
}
}

64
cl_search_api/src/main/java/com/bfd/mf/common/util/OperatorUtil.java
View File

@ -1,64 +0,0 @@
package com.bfd.mf.common.util;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
public class OperatorUtil {
private static final Character[] seps = new Character[] { '&', '|', '~', '<', '《', '>', '》', '(', '（', ')', '）',
',', '，', '"', '“', '”' };// 分隔符,将整个表达式分隔成若干个子串进行处理
private static final Character[] ops = new Character[] { '&', '|', '~' };// 去掉'!'
public static final List<Character> operators = new ArrayList<Character>();
public static final List<Character> seperators = new ArrayList<Character>();
static {
operators.addAll(Arrays.asList(ops));
seperators.addAll(Arrays.asList(seps));
}
public static boolean isOperator(char c) {
return operators.contains(c);
}
public static boolean isSeperator(char c) {
return seperators.contains(c);
}
public static int priorityCompare(char op1, char op2) {
// ()>!>&>|
switch (op1) {
case '!':
case '！':
return op2 == '(' || op2 == ')' ? -1 : 1;
case '&':
return op2 == '!' ? -1 : (op2 == '|' ? 1 : 0);
case '|':
return op2 == '|' ? 0 : -1;
}
return 0;
}
public static boolean matchChar(char c, char c2) {
switch (c) {
case '(':
case '（':
return c2 == ')' || c2 == '）';
case ')':
case '）':
return c2 == '（' || c2 == '(';
case '>':
case '》':
return c2 == '<' || c2 == '《';
case '<':
case '《':
return c2 == '>' || c2 == '》';
default:
break;
}
return false;
}
}

354
cl_search_api/src/main/java/com/bfd/mf/common/util/analysis/DataAnalysisUtils.java
View File

@ -1,354 +0,0 @@
package com.bfd.mf.common.util.analysis;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.mf.common.util.enums.SpreadEnums;
import com.bfd.mf.common.web.vo.view.analysis.*;
import com.bfd.mf.common.web.vo.view.monitor.ESMonitorEntity;
import com.bfd.nlp.common.util.object.TObjectUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.stream.IntStream;
public class DataAnalysisUtils {
private static final Logger logger = LoggerFactory.getLogger(DataAnalysisUtils.class);
private static final Long ONE_HOUR = 60 * 60 * 1000L;
private static final Integer COUNT = 10;
private static final Integer ZERO = 0;
private static final Long ONE_DAY = 60 * 60 * 24 * 1000L;
private static final Long EIGHT_HOUR = 60 * 60 * 8 * 1000L;
/**
* 根据时间，敏感等趋势图得到 pie 和 four total Count
*/
public static JSONObject fetchPieAndCountNumberByLineJson(JSONObject jsonObject) {
Long negativeNumber = 0L;
try {
Map<String, List<Long>> emotionsMap = (Map<String, List<Long>>) jsonObject.get("emotions");
List<Long> negativeTimeList = emotionsMap.get(ESConstant.NEGATIVE_CN_TAG);
for (Long number : negativeTimeList) {
negativeNumber = negativeNumber + number;
}
PieCount emotionPieCount = fetchPieCount(emotionsMap, 0);
Map<String, List<Long>> channelsMap = (Map<String, List<Long>>) jsonObject.get("channels");
PieCount channelPieCount = fetchPieCount(channelsMap, 1);
jsonObject.put("emotionCounts", emotionPieCount.getEmotionCounts());
jsonObject.put("channelCounts", channelPieCount.getChannelCounts());
jsonObject.put("negativeNumber", negativeNumber);
} catch (Exception e) {
logger.error("fetchPieAndCountNumberByLineJson error =", e);
}
return jsonObject;
}
private static PieCount fetchPieCount(Map<String, List<Long>> channelsMap, int type) {
PieCount pieCount = new PieCount();
if (TObjectUtils.nonNull(channelsMap)) {
for (String name : channelsMap.keySet()) {
long temp;
List<Long> longs = channelsMap.get(name);
temp = longs.stream().mapToLong(number -> number).sum();
DataPieCount dataPieCount = new DataPieCount(name, temp);
if (type == 0) {
pieCount.getEmotionCounts().add(dataPieCount);
} else if (type == 1) {
pieCount.getChannelCounts().add(dataPieCount);
}
}
}
return pieCount;
}
public static Long[] getTimeList(Long startTime, Long endTime, Integer type) {
Long[] timeList = null;
try {
Long interval;
if (type.equals(SpreadEnums.TREND.UNIT_HOUR)) {
interval = ONE_HOUR;
} else if (type.equals(SpreadEnums.TREND.UNIT_DAY)) {
interval = ONE_DAY;
} else {
throw new Exception(" !!! type is illegal !!!");
}
Double intervalNum = Math.ceil((endTime - startTime) / (double) interval);
if ((endTime - startTime) % interval == 0) {
intervalNum = intervalNum + 1;
}
timeList = new Long[intervalNum.intValue()];
Long time = startTime;
for (int i = 0; i < intervalNum; i++) {
if (i == 0) {
timeList[i] = time;
} else if (i == intervalNum - 1) {
timeList[i] = endTime;
} else {
time = time + interval;
Long dayStartTime = DateTrendUtils.getDayStartTime(time);
if (dayStartTime <= timeList[i - 1]) {
timeList[i] = time;
} else {
timeList[i] = dayStartTime;
}
}
}
} catch (Exception e) {
logger.error("[DataAnalysisUtils] getTimeList: failed, startTime is {}, endTime is {}, timeList is {}, error is ", startTime, endTime, timeList, e);
}
return timeList;
}
public static Map<String, List<DataLineCount>> initEmotionList(Long[] timeList) {
Map<String, List<DataLineCount>> negativeChannelMaps = new HashMap<>();
List<String> emotionName = new ArrayList<>();
emotionName.add(ESConstant.NEGATIVE_TAG);
emotionName.add(ESConstant.COMMON_TAG);
for (String name : emotionName) {
for (Long aTimeList : timeList) {
List<DataLineCount> dataLineCounts = new ArrayList<>();
DataLineCount dataLineCount = fetchDataLineCountObject(0, name, aTimeList);
dataLineCounts.add(dataLineCount);
negativeChannelMaps.put(name, dataLineCounts);
}
}
return negativeChannelMaps;
}
private static DataLineCount fetchDataLineCountObject(long data, String name, Long aStart) {
DataLineCount dataLineCount = new DataLineCount();
dataLineCount.setData(data);
dataLineCount.setName(name);
dataLineCount.setTimestamp(aStart);
return dataLineCount;
}
public static void parseTimeSeries(Map<String, List<DataLineCount>> channelMaps,
Map<String, List<DataLineCount>> emotionMaps,
String emotionTag,
Map<String, List<DataLineCount>> combineChannelMaps) {
for (Map.Entry<String, List<DataLineCount>> entry : channelMaps.entrySet()) {
List<DataLineCount> commonChannelLists = entry.getValue();
if (Optional.ofNullable(commonChannelLists).isPresent()) {
for (DataLineCount dataLineCount : commonChannelLists) {
if (emotionTag.equals(ESConstant.NEGATIVE_TAG)) {
buildEmotionResult(emotionMaps, dataLineCount, "0.0-" + ESConstant.SENTIMENTAL_THRESHOLD);
} else if (emotionTag.equals(ESConstant.NEUTER_TAG)) {
buildEmotionResult(emotionMaps, dataLineCount, ESConstant.SENTIMENTAL_THRESHOLD + "-" + ESConstant.SENTIMENTAL_MID);
} else {
buildEmotionResult(emotionMaps, dataLineCount, ESConstant.SENTIMENTAL_MID + "-" + ESConstant.SENTIMENTAL_MAX);
}
if (null != combineChannelMaps) {
if (!combineChannelMaps.containsKey(entry.getKey())) {
List<DataLineCount> combineChannelLists = new ArrayList<>();
DataLineCount newDataLineCount = new DataLineCount();
newDataLineCount.setName(dataLineCount.getName());
newDataLineCount.setTimestamp(dataLineCount.getTimestamp());
newDataLineCount.setData(dataLineCount.getData());
combineChannelLists.add(newDataLineCount);
combineChannelMaps.put(entry.getKey(), combineChannelLists);
} else {
List<DataLineCount> alreadyLists = combineChannelMaps.get(entry.getKey());
if (alreadyLists.contains(dataLineCount)) {
alreadyLists.get(alreadyLists.indexOf(dataLineCount)).setData(alreadyLists.get(alreadyLists.indexOf(dataLineCount)).getData() + dataLineCount.getData());
} else {
alreadyLists.add(dataLineCount);
}
}
}
}
}
}
}
/**
* 计算情感
*/
private static void buildEmotionResult(Map<String, List<DataLineCount>> emotionMaps, DataLineCount dataLineCount, String emotionTag) {
if (!emotionMaps.containsKey(emotionTag)) {
List<DataLineCount> negativeLists = new ArrayList<>();
DataLineCount newDataLineCount = new DataLineCount();
newDataLineCount.setName(dataLineCount.getName());
newDataLineCount.setTimestamp(dataLineCount.getTimestamp());
newDataLineCount.setData(dataLineCount.getData());
negativeLists.add(newDataLineCount);
emotionMaps.put(emotionTag, negativeLists);
} else {
List<DataLineCount> alreadyLists = emotionMaps.get(emotionTag);
if (alreadyLists.contains(dataLineCount)) {
alreadyLists.get(alreadyLists.indexOf(dataLineCount)).setData(alreadyLists.get(alreadyLists.indexOf(dataLineCount)).getData() + dataLineCount.getData());
} else {
DataLineCount newDataLineCount = new DataLineCount();
newDataLineCount.setName(dataLineCount.getName());
newDataLineCount.setTimestamp(dataLineCount.getTimestamp());
newDataLineCount.setData(dataLineCount.getData());
alreadyLists.add(newDataLineCount);
}
}
}
public static Map<String, List<DataLineCount>> fetchPerChannelCountByListData(List<ESMonitorEntity> cacheEsMonitorEntityList,
Long[] timeList, int type) {
Map<String, List<DataLineCount>> channelMaps = new HashMap<>();
for (ESMonitorEntity esMonitorEntity : cacheEsMonitorEntityList) {
String name = esMonitorEntity.getDocType();
DataAnalysisUtils.fetchFilterResultByTimeCompare(channelMaps, name, esMonitorEntity, timeList, type);
}
return channelMaps;
}
/**
* 时间段统计数量
*/
public static void fetchFilterResultByTimeCompare(Map<String, List<DataLineCount>> negativeChannelMaps,
String name,
ESMonitorEntity esMonitorEntity,
Long[] timeList,
int type) {
Long pubTime = esMonitorEntity.getPubTime();
int length = timeList.length;
boolean isZeroTime = false;
for (int i = 0; i < length; i++) {
Long aStart;
Long aEnd = null;
if (type == 1) {
// 针对天
if (i == length - 1) {
long temp;
aStart = timeList[i];
isZeroTime = isZeroTime(aStart);
if (isZeroTime) {
aEnd = timeList[i];
} else {
temp = getDayTime(aStart);
if (aStart < temp) {
// 大于8小时
if (temp > aStart) {
temp = temp - ONE_DAY;
}
aEnd = aStart;
aStart = temp;
} else {
aEnd = aStart;
aStart = temp;
}
}
} else if (i == length - 2) {
long temp;
aStart = timeList[i];
temp = getDayTime(aStart);
if (aStart <= temp) {
aEnd = aStart + ONE_DAY;
}
} else {
aStart = timeList[i];
aEnd = timeList[i + 1];
}
} else {
if (i == length - 1) {
aStart = timeList[i];
aEnd = aStart + ONE_DAY;
} else {
aStart = timeList[i];
aEnd = timeList[i + 1];
}
}
if (pubTime >= aStart && (pubTime < aEnd || isZeroTime)) {
long temp = aStart;
// 天，零点，
if (i == length - 1 && !isZeroTime && type == 1) {
temp = aEnd;
}
if (negativeChannelMaps.containsKey(name)) {
List<DataLineCount> dataLineCounts = negativeChannelMaps.get(name);
boolean existTime = false;
for (DataLineCount dataLineCount : dataLineCounts) {
Long timestamp = dataLineCount.getTimestamp();
if (Objects.equals(timestamp, temp)) {
dataLineCount.setData(dataLineCount.getData() + 1);
existTime = true;
break;
}
}
if (!existTime) {
DataLineCount dataLineCount = fetchDataLineCountObject(1L, name, temp);
dataLineCounts.add(dataLineCount);
negativeChannelMaps.put(name, dataLineCounts);
}
} else {
List<DataLineCount> dataLineCounts = new ArrayList<>();
DataLineCount dataLineCount = fetchDataLineCountObject(1L, name, temp);
dataLineCounts.add(dataLineCount);
negativeChannelMaps.put(name, dataLineCounts);
}
break;
}
}
}
private static boolean isZeroTime(Long time) {
SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss");
return dateFormat.format(time).equals("00:00:00");
}
private static long getDayTime(Long aStart) {
long dayTime = aStart - (aStart) % ONE_DAY;
dayTime = dayTime - EIGHT_HOUR;
dayTime = dayTime + ONE_DAY;
return dayTime;
}
public static Long[] fetchPerTimeByPerChannelCountResult(Map<String, List<Long>> channelPerTimeMap, Long[] timeArray) {
Long[] docNumList = IntStream.range(0, timeArray.length).mapToObj(i -> 0L).toArray(Long[]::new);
for (int i = 0; i < timeArray.length; i++) {
long temp = 0;
for (String key : channelPerTimeMap.keySet()) {
temp = channelPerTimeMap.get(key).get(i) + temp;
}
docNumList[i] = temp;
}
return docNumList;
}
public static JSONArray getEmotionOrChannelTrendTime(List<Long> dataList,
Long[] timeList,
JSONArray timeArr,
int type) throws ParseException {
if (TObjectUtils.isListEmpty(dataList)) {
return null;
}
Long[] docNumList = new Long[dataList.size()];
for (int i = 0; i < dataList.size(); i++) {
docNumList[i] = dataList.get(i);
}
JSONArray countByHourData = null;
switch (type) {
case 1:
countByHourData = DateTrendUtils.getDayCountByHourData(timeList, timeArr, docNumList);
break;
case 2:
countByHourData = DateTrendUtils.getMonthCountByHourData(timeList, timeArr, docNumList);
break;
case 3:
countByHourData = DateTrendUtils.getYearCountByHourData(timeList, timeArr, docNumList);
default:
break;
}
return countByHourData;
}
}

279
cl_search_api/src/main/java/com/bfd/mf/common/util/analysis/DateTrendUtils.java
View File

@ -1,279 +0,0 @@
package com.bfd.mf.common.util.analysis;
import com.alibaba.fastjson.JSONArray;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.GregorianCalendar;
import java.util.LinkedHashSet;
import java.util.Set;
public class DateTrendUtils {
/**
* 获得该月第一天
*/
public static String getFirstDayOfMonth(Integer year, Integer month) throws ParseException {
Calendar cal = Calendar.getInstance();
//设置年份
cal.set(Calendar.YEAR, year);
//设置月份
cal.set(Calendar.MONTH, month - 1);
//获取某月最小天数
int firstDay = cal.getActualMinimum(Calendar.DAY_OF_MONTH);
//设置日历中月份的最小天数
cal.set(Calendar.DAY_OF_MONTH, firstDay);
//格式化日期
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
return sdf.format(cal.getTime());
}
/**
* 获得该月最后一天
*/
private static String getLastDayOfMonth(Integer year, Integer month) throws ParseException {
Calendar cal = Calendar.getInstance();
//设置年份
cal.set(Calendar.YEAR, year);
//设置月份
cal.set(Calendar.MONTH, month - 1);
//获取某月最大天数
int lastDay = cal.getActualMaximum(Calendar.DAY_OF_MONTH);
//设置日历中月份的最大天数
cal.set(Calendar.DAY_OF_MONTH, lastDay);
//格式化日期
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
return sdf.format(cal.getTime());
}
/**
* 得到月的统计值
*/
public static JSONArray getMonthCountByHourData(Long[] timeList, JSONArray monthArr, Long[] countArr) throws ParseException {
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM");
// 存储 Mark
JSONArray markArr = new JSONArray();
for (int i = 0; i < monthArr.size(); i++) {
Long dayEndTime = getMonthEndDay(getDayEndTime(format.parse(monthArr.getString(i)).getTime()));
int mark = 0;
for (int j = 0; j < timeList.length; j++) {
if (dayEndTime < timeList[j]) {
mark = j;
break;
} else if (dayEndTime.equals(timeList[j])) {
mark = j + 1;
break;
}
}
mark = mark == 0 ? timeList.length : mark;
markArr.add(mark);
}
// new data
return getNewCountData(markArr, countArr);
}
/**
* 得到天的统计值
*/
public static JSONArray getDayCountByHourData(Long[] timeList, JSONArray dayArr, Long[] countArr) throws ParseException {
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
// 存储 Mark
JSONArray markArr = new JSONArray();
for (int i = 0; i < dayArr.size(); i++) {
Long dayEndTime = getDayEndTime(format.parse(dayArr.getString(i)).getTime());
int mark = 0;
for (int j = 0; j < timeList.length; j++) {
if (dayEndTime < timeList[j]) {
mark = j;
break;
} else if (dayEndTime.equals(timeList[j])) {
mark = j + 1;
break;
}
}
mark = mark == 0 ? timeList.length : mark;
markArr.add(mark);
}
// new data
return getNewCountData(markArr, countArr);
}
/**
* 得到年的统计值
*/
public static JSONArray getYearCountByHourData(Long[] timeList, JSONArray yearArr, Long[] countArr) throws ParseException {
SimpleDateFormat format = new SimpleDateFormat("yyyy");
// 存储 Mark
JSONArray markArr = new JSONArray();
for (int i = 0; i < yearArr.size(); i++) {
Long dayEndTime = getDayEndTime(getYearEndDay(format.parse(yearArr.getString(i)).getTime()));
int mark = 0;
for (int j = 0; j < timeList.length; j++) {
if (dayEndTime < timeList[j]) {
mark = j;
break;
} else if (dayEndTime.equals(timeList[j])) {
mark = j + 1;
break;
}
}
mark = mark == 0 ? timeList.length : mark;
markArr.add(mark);
}
// new data
return getNewCountData(markArr, countArr);
}
/**
* 计算得到的统计值
*/
private static JSONArray getNewCountData(JSONArray markArr, Long[] countArr) {
JSONArray newData = new JSONArray();
for (int j = 0; j < markArr.size(); j++) {
int startIndex = 0;
int endIndex;
if (j == 0) {
endIndex = markArr.getInteger(j);
} else {
endIndex = markArr.getInteger(j);
startIndex = markArr.getInteger(j - 1);
}
long count = 0;
for (; startIndex < endIndex; startIndex++) {
count = count + countArr[startIndex];
}
newData.add(count);
}
return newData;
}
/**
* 一天的开始时间
*/
public static Long getDayStartTime(Long time) {
Calendar todayStart = new GregorianCalendar();
todayStart.setTimeInMillis(time);
todayStart.set(Calendar.HOUR_OF_DAY, 0);
todayStart.set(Calendar.MINUTE, 0);
todayStart.set(Calendar.SECOND, 0);
todayStart.set(Calendar.MILLISECOND, 0);
return todayStart.getTime().getTime();
}
/**
* 一天的结束时间
*/
public static Long getDayEndTime(Long time) {
Calendar todayEnd = new GregorianCalendar();
todayEnd.setTimeInMillis(time);
todayEnd.set(Calendar.HOUR_OF_DAY, 23);
todayEnd.set(Calendar.MINUTE, 59);
todayEnd.set(Calendar.SECOND, 59);
todayEnd.set(Calendar.MILLISECOND, 999);
return todayEnd.getTime().getTime();
}
/**
* 月的最后一天
*/
private static Long getMonthEndDay(Long time) {
Calendar ca = new GregorianCalendar();
ca.setTimeInMillis(time);
ca.set(Calendar.DAY_OF_MONTH, ca.getActualMaximum(Calendar.DAY_OF_MONTH));
return ca.getTime().getTime();
}
/**
* 月的第一天
*/
public static Long getMonthStartDay(Long time) {
Calendar c = new GregorianCalendar();
c.setTimeInMillis(time);
c.add(Calendar.MONTH, 0);
c.set(Calendar.DAY_OF_MONTH, 1); //设置为1号,当前日期既为本月第一天
return c.getTime().getTime();
}
/**
* 年的最后一天
*/
private static Long getYearEndDay(Long time) {
Calendar c = new GregorianCalendar();
c.setTimeInMillis(time);
c.set(Calendar.DAY_OF_YEAR, c.getActualMaximum(Calendar.DAY_OF_YEAR));
return c.getTime().getTime();
}
/**
* 年的第一天
*/
public static Long getYearStartDay(Long time) {
Calendar c = new GregorianCalendar();
c.setTimeInMillis(time);
c.add(Calendar.YEAR, 0);
c.set(Calendar.DAY_OF_YEAR, 1);
return c.getTime().getTime();
}
/**
* 得到天，月，年数组
*/
private static JSONArray getStrTime(Long[] timeList, SimpleDateFormat format) {
Set<String> setTime = new LinkedHashSet<>();
for (Long aLong : timeList) {
String s = format.format(aLong);
setTime.add(s);
}
JSONArray arrayTime = new JSONArray();
arrayTime.addAll(setTime);
return arrayTime;
}
/**
* 转换 day,month,year arr
*/
public static JSONArray convertStrTime(Long[] timeList, int type) {
JSONArray arrayTime;
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
switch (type) {
case 1:
// day
format = new SimpleDateFormat("yyyy-MM-dd");
break;
case 2:
// month
format = new SimpleDateFormat("yyyy-MM");
break;
case 3:
// year
format = new SimpleDateFormat("yyyy");
break;
default:
break;
}
arrayTime = getStrTime(timeList, format);
return arrayTime;
}
/**
* 获取小时
*/
public static String getHourByLongTime(Long longTime) {
SimpleDateFormat format = new SimpleDateFormat("HH");
return format.format(longTime);
}
}

43
cl_search_api/src/main/java/com/bfd/mf/common/util/cache/ResponseParseUtils.java
View File

@ -1,43 +0,0 @@
package com.bfd.mf.common.util.cache;
import com.bfd.mf.common.util.es.MonitorUtils;
import com.bfd.mf.common.web.vo.view.monitor.ESMonitorBaseEntity;
import com.bfd.nlp.common.util.object.TObjectUtils;
import org.elasticsearch.action.search.SearchResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class ResponseParseUtils {
private static Logger logger = LoggerFactory.getLogger(ResponseParseUtils.class);
public static long combineEsAndCatchResponse(
List<SearchResponse> cacheResponse,
Map<String, ESMonitorBaseEntity> timeSeriesMap) {
Map<String, Integer> idSet = combineEsAndCatchResponseWithDocId(
cacheResponse,
timeSeriesMap);
return idSet.size();
}
public static Map<String, Integer> combineEsAndCatchResponseWithDocId(
List<SearchResponse> cacheResponse,
Map<String, ESMonitorBaseEntity> timeSeriesMap) {
Map<String, Integer> docIdWithAreaMap = new HashMap<>();
if (!TObjectUtils.isListEmpty(cacheResponse)) {
cacheResponse.forEach(response ->
MonitorUtils.parseTimeSeries(
response,
docIdWithAreaMap,
timeSeriesMap)
);
}
logger.info("[combineEsAndCatchResponseWithDocIdCrawl method] the all doc number is {},the folder doc number is {}", docIdWithAreaMap.size(), timeSeriesMap.size());
return docIdWithAreaMap;
}
}

36
cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ConditionCommon.java
View File

@ -1,36 +0,0 @@
package com.bfd.mf.common.util.constants;
public class ConditionCommon {
public static final String SPLIT = "_";
public static final Integer HOUR = 0;
public static final Integer DAY = 1;
public static final Integer MONTH = 2;
public static final Integer YEAR = 3;
public static final String PROVINCE = "province";
public static final String CITY = "city";
// other
public static final String MEDIA_COUNTS = "mediaCounts";
public static final String MEDIA_COUNTS_ALL = "mediaCountsALL";
public static final String HOUR_COUNT_MAP = "hourCountMap";
public static final String WORD_CLOUD = "wordCloud";
public static final String PLACE_CLOUD = "placesCloud";
public static final String HASH_TAG_CLOUD = "hashTagCloud";
public static final String EMOJI_CLOUD = "emojiCloud";
public static final String OPINION_CLOUD = "opinionsCloud";
// weibo
public static final String VERIFIED_COUNTS = "verifiedCounts";
public static final String SEX_COUNTS = "sexCounts";
public static final String AGE_COUNT_MAP = "ageCountMap";
public static final String WEIBO_COUNT = "weiboCount";
public static final String NORMAL = "常规";
public static final String PROMOTION = "推广";
}

663
cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ESConstant.java
View File

@ -1,663 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.util.constants;
import com.bfd.mf.common.util.utility.EsQueryConditionUtils;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class ESConstant {
public static final String MEDIA_AREA_KEY = "dict";
public static final float BOOST_WEIGHT = 10f;
// 相似查询返回的最大数据量
public static final Integer MAX_RESULT_WINDOWS_SIZE = 10000;
public static final Integer MAX_SCROLL_SIZE = 5000;
public static final Integer MAX_SCORLL_SIZE_CL = 2000;
public static final Long MAX_SEARCH_TIME_MINUTES = 2L;
public static final String SUBJECT_CLUSTER_INDEX_PREFIX = "mf_subject";
public static final String ES_KEY = "es";
public static final String ES_NORMAL_KEY = "es_normal";
public static final String CACHE_KEY = "cache";
public static final int BULK_SIZE = 100;
public static final String INDEX_TYPE = "docs";
public static final String bigramPostfix = ".shingles";
public static final String bigramKey = "bigram";
public static final String filterTypeKey = "filter_type";
public static final String queryTypeKey = "query_type";
public static final Double aggOffset = 16 * 6
* 60 * 1000d;
public static final Double ONE_DAY = 24 * 60 * 60 * 1000d;
public static final Double ONE_HOUR = 60 * 60 * 1000d;
public static final String SENTIMENT = "sentiment";
public static final String SUMMARY = "summary";
public static final String FILEPATH = "filePath";
public static final String ISDOWNLOAD = "isDownload";
public static final String VIDEOURL = "videoUrl";
/**
* *
*
* @FIXED 2017.11.13 increase the runtime,from 3 min to 6 min
*/
public static final long THREAD_TIME = 15 * 60 * 1000L;
public static String DESC = "desc";
public static String ASC = "asc";
/**
* pubTime时间由远及近
*/
public static String DEFUALT_SORT = "createdTimeDesc";
/**
* pubTime时间由远及近
*/
public static String PUBTIME_ASC_SORT_FLAG = "timeAsc";
/**
* 评论数：commentDescFlag
发帖时间:（近-远）timeDesc
发帖时间：(远-近) timeAsc
采集时间：crawlTimeDesc
*/
/**
* pubTime时间由近及远
*/
public static String PUBTIME_DESC_SORT_FLAG = "timeDesc";
public static String CRAWLTIME_DESC_SORT_FLAG = "crawlTimeDesc";
/**
* createTime时间由近及远
*/
public static String CREATE_TIME_DESC_SORT_FLAG = "createdTimeDesc";
/**
* createTime时间由远及近
*/
public static String CREATE_TIME_ASC_SORT_FLAG = "createdTimeAsc";
/**
* 重要度排序
*/
public static String SCORE_DESC_SORT_FLAG = "scoreDesc";
/**
* Alexa
*/
public static String ALEXA_ASC_SORT_FLAG = "alexaAsc";
/***
*
*/
public static Float ALEXA_DEFAULT_SCORE = 1000000.00F;
/**
* 转载量排序
*/
public static String FOLDER_NUMBER_DESC_SORT_FLAG = "folderNumberDescSortFlag";
/**
* 综合排序：1:转载量排序;2:转载量一样按照得分排序;3:随后按照pubTime时间倒序
*/
public static String REPORT_SORT_FLAG = "reportSortFlag";
public static String SCORE = "_score";
public static String SCORE_FROM_SOURCE = "score";
/**
* 关键词前缀
*/
public static final String TAG_START = "<em>";
/**
* 关键词后缀
*/
public static final String TAG_END = "</em>";
/**
* 标题关键词分隔符
*/
public static final String TITLE_KEYWORDS_SPLIT_SYSBOL = ",";
/**
* 相关度等级词分隔符
*/
public static final String RELATEDWORDS_HIGH = "high";
public static final String RELATEDWORDS_MEDIUM = "medium";
public static final String RELATEDWORDS_LOW = "low";
/**
* 相关度分级
*/
public static final float RELATEDRATE_MIN = 0.00F;
public static final float RELATEDRATE_MAX = 1.00F;
public static final int RELATEDRATE_MIN_SEARCH_NUMBER = 10;
/**
* 排除关键词分隔符
*/
public static final String EXCLUDE_KEYWORDS_SPLIT_SYSBOL = ",";
public static final String EXCLUDE_KEYWORDS_SPLIT_CN_SYSBOL = "，";
/**
* 最小匹配度
*/
public static final Integer MINI_NUMBER_SHOULD_MATCH = 1;
/**
* 最大摘要长度
*/
public static final Integer MAX_FRAGMENT_SIZE = 10000;
/**
* 数据监控页面摘要长度
*/
public static final Integer MONITOR_FRAGMENT_SIZE = 110;
// 报告摘要长度
public static final Integer Report_FRAGMENT_SIZE = 100;
public static final Integer Foreign_Report_FRAGMENT_SIZE = 200;
/**
* 数据监控页面标题高亮长度
*/
public static final Integer MONITOR_FRAGMENT_TITLE_SIZE = 50;
/**
* 数据导出查询大集群分页查询，每次最大查询量
*/
public static final Integer EXPORT_SCROLL_SIZE = 1000;
/**
* 自定义长度
*/
// title 长度
public static final Integer MONITOR_FRAGMENT_USER_DEFINED_TITLE_SIZE = 30;
// 系统报告自定义长度 40
public static final Integer SYS_REPORT_FRAGMENT_USER_DEFINED_TITLE_SIZE = 40;
public static final Integer FOREIGN_SYS_REPORT_FRAGMENT_USER_DEFINED_TITLE_SIZE = 80;
public static final Integer MAX_FRAGMENT_USER_DEFINED_TITLE_SIZE = 50;
// 预警报告的摘要长度
public static final Integer MONITOR_FRAGMENT_USER_DEFINED_DIGEST_SIZE = 150;
public static final String DEFAULT_JOIN_SYMBOL = ".";
/**
* 主体情感默认名称
*/
public static final String DEFAULT_ENTRY_NAME = "default";
public static final String EMOTION_ENTRY = "emotionEntry";
public static final String EMOTION_VALUE = "emotionValue";
public static final String EMOTION_NAME = "entryName";
public static final String OFF_SET = "offSet";
public static final String MINIMUM_SHOULD_MATCH = "1";
/*
* docType(news\bbs...):int 类型常量
*/
public static final int DOCTYPENEWS = 0;
public static final int DOCTYPEBBS = 1;
/*
* 组合查询语句key值类型
*/
public static final String KEYTYPEOPERA = "operator";
public static final String KEYTYPEWORD = "word";
/*
* 数据来源渠道 (hl \ bfd)
*/
public static final String CHANNELHL = "HL";
public static final String CHANNELBFD = "BFD";
/*
*1:type:content、title、all
*2: text:words(目前要是关键词列表)
*/
public static final String SEARCH_SCOPE_TYPE_TITLE = "title";
public static final String ICON = "icon";
public static final String SEARCH_SCOPE_TYPE_CONTENT = "content";
//标题+正文
public static final String SEARCH_SCOPE_TYPE_TITLE_ALL = "title_all";
public static final String SEARCH_SCOPE_TYPE_AUTHOR = "author";
public static final String SEARCH_SCOPE_TYPE_FORWARD = "forwardContent";
public static final String SEARCH_SCOPE_TYPE_TITLE_ORIGIN = "title_origin";
public static final String SEARCH_SCOPE_TYPE_CONTENT_ORIGIN = "content_origin";
public static final String SEARCH_SCOPE_TYPE_ALL = "all";
public static final String SEARCH_SCOPE_WECHAT_SUBSCRIPTION = "wechat_subscription";
/**
* SQ 4.0新增 UGC EGC PGC
*/
public static final String SEARCH_SCOPE_UGC = "ugc";
public static final String SEARCH_SCOPE_EGC = "egc";
public static final String SEARCH_SCOPE_PGC = "pgc";
/**
* 关键词
*/
public static final String SEARCH_SCOPE_TYPE_KEYWORDS = "keyWords";
public static final String REL_TYPE = "relType";
public static final String PRIMARY_POST = "primaryPost";
public static final String PRIMARY = "primary";
public static final String PT_PRIMARY = "ptPrimary";
public static final String DT_PRIMARY = "dtPrimary";
public static final String PT_ALL = "ptAll.keyword";
public static final String DT_ALL = "dtAll.keyword";
public static final String DATA_COUNT = "dataCount";
/**
* 回溯开始时间
*/
public static final String CON_START_TIME = "conStartTime";
/**
* 回溯结束时间
*/
public static final String CON_END_TIME = "conEndTime";
public static final String SUBJECT_ID = "subjectId";
/*
*1:非敏感 positive
*2:敏感 negative
*/
public static final String COMMON_TAG = "common";
public static final String NEGATIVE_TAG = "negative";
public static final String NEUTER_TAG = "neuter";
public static final Integer COMMON_TAG_VALUE = 1;
public static final Integer NEGATIVE_TAG_VALUE = 3;
public static final Integer NEUTER_TAG_VALUE = 2;
public static final String COMMON_CN_TAG = "正面";
public static final String NEGATIVE_CN_TAG = "负面";
public static final String NEUTER_CN_TAG = "中性";
public static final String AVAILABILITY = "availability";
public static final List<String> SENTIMENT_LIST = new ArrayList<String>();
public static String URL_HASH = "urlHash";
static {
SENTIMENT_LIST.add(COMMON_TAG);
SENTIMENT_LIST.add(NEGATIVE_TAG);
SENTIMENT_LIST.add(NEUTER_TAG);
}
// 正面 [0.8-1]
// 中性（0.2-0.8)
// 负面 [0-0.2]
public static final Double SENTIMENTAL_MID = 0.8;
public static final Double SENTIMENTAL_THRESHOLD = 0.2;
public static final Double SENTIMENTAL_MIN = 0.0;
public static final Double SENTIMENTAL_MAX = 1.0;
public static final String OPERATOR_OR = "or";
public static final String OPERATOR_EQUAL = "=";
public static final String OPERATOR_SPACE = " ";
public static final String OPERATOR_ALL = "*:*";
public static final String OPERATOR_CONNENT_ALL = ":";
public static final String OPERATOR_SINGLE_ALL = "*";
/*
*define es mapping fields
*/
public static String PUBTIME = "pubTime";
public static String CHANNEL_HYLANDA = "hylanda";
public static String PUBTIME_STR = "pubTimeStr";
public static String CREATETIME = "createTime";
public static String CREATETIMESTR = "createTimeStr";
public static String CREATETIMEDAY = "createDay";
public static String PUTTIME = "putTime";
public static String CRAWLTIME = "crawlTime";
public static String ORIGINAL_SOURCE = "originalSource";
public static String CONTENT_SIMHASH = "contentSimHash";
public static String QUOTE_COUNT = "quoteCount";
/**
* 内容
*/
public static String CONTENT_TAG = "contentTag";
public static String DT_EN_2 = "dtEn2";
public static String DT_EN_1 = "dtEn1";
public static String COMMENTS_COUNT = "commentsCount";
public static String PAGE_COMMENTS_COUNT = "pageCommentCount";
public static String ATTITUDES_COUNT = "attitudesCount";
public static String TITLE_SIMHASH = "titleSimHash";
public static String ORIGINAL_CONTENT_SIMHASH = "originalContentSimHash";
public static String ORIGINAL_TITLE_SIMHASH = "originalTitleSimHash";
public static String DOC_TYPE = "docType";
public static String CHINA_DOC_TYPE = "chinaDocType";
public static String SYS_SENTIMENT_TAG = "sysSentimentTag";
public static final String TITLE = "title";
public static final String USER_LEVEL_NAME = "userLevelName";
public static String SOURCE = "source";
public static String CRAWLDATAFLAG= "crawlDataFlag";
//微信专属字段，内容固定 value="微信"
public static String WEIXIN_SOURCE = "weixinSource";
public static String CHANNEL = "channel";
public static final String CONTENT = "content";
public static final String SYS_SENTIMENT = "sysSentiment";
public static String POST_SOURCE = "postSource";
/**
* DT情感标签
*/
public static final String DT_SENTIMENT = "dtSentiment";
/**
* PT情感标签
*/
public static final String PT_SENTIMENT = "ptSentiment";
public static String URL = "url";
public static String DOC_ID = "docId";
public static String DATA_ID = "dataId";
public static String _ID = "_id_";
public static String SYS_ABSTRACT = "sysAbstract";
public static String SYS_KEYWORDS = "sysKeywords";
// 所有热词
public static String HL_KEYWORDS = "hlKeywords";
public static String ALL_KEYWORDS = "allKeywords";
public static String COMMENT_KEY_WORDS = "commentKeywords";
public static String OPTIONS = "opinions";
// 地名：places
public static String PLACES = "places";
// 微博话题：hashTag
public static String HASH_TAG = "hashTag";
public static String PICTURE_LIST = "pictureList";
// 表情：expression
public static String EXPRESSION = "expression";
// 评价：opinions
public static String OPINIONS = "opinions";
public static String WEIBO_AUTHOR = "author";
public static String WEIBO_USER_ID = "userId";
public static String AUTHOR = "author";
public static String WEIXIN_NAME = "weixinName";
public static final String CHANNEL_WEIXIN = "weixin";
public static final String HL_DOCID = "hylandaDocId";
public static final String LANGUAGE = "language";
public static String AREA = "area";
/**
* _all字段
*/
public static String _ALL = "_all";
/**
* 口碑
*/
public static final String SHOP_MD5 = "shopUrlHash";
public static final String HAS_PICTURE = "isCommentImg";
public static final String HAS_CONTENT = "isComment";
public static final String CUSTOMER_LEVEL = "customerLevel";
public static final String COMMENT_STAR_LEVEL = "starLevel";
public static final String SHOP_NAME = "shopName";
public static final String NICKNAME = "nickName";
public static final String COMMENT_SEARCH_ALL = "comment_all";
public static final String USER_AVATAR = "userAvatar";
public static final String UNIQUE_ID = "uniqueId";
public static final String SHOP_URL = "shopUrl";
public static final String USER_URL = "userUrl";
public static final String CONTRIBUTION = "contribution";
public static final String SCORE_FACTOR = "scoreFactor";
public static final String FACTOR_KEY = "factorKey";
public static final String FACTOR_VALUE = "factorValue";
public static final String CONTENT_SIZE = "commentLength";
public static final String SHOP_LABEL = "label";
public static final String SHOP_CITY = "shopCity";
public static final String SHOP_PROVINCE = "shopProvince";
public static final String USER_CITY = "userCity";
public static final String COUNTRY = "country";
public static final String USER_PROVINCE = "userProvince";
public static final String SHOP_LABELS = "shopLabels";
public static final String COMMENT_IMG = "commentImg";
/**
* 用户头像
*/
public static String WEIBO_AVATOR = "avatar";
public static final String SOCIAL = "social";
public static final String BBS = "bbs";//论坛
public static final String WEI_BO = "weibo";//微博
public static final String WEI_XIN = "weixin";//微信
public static final String CH_WEI_XIN = "微信";
public static final String BLOG = "blog";//博客
public static final String NEWS = "news";//网站
public static final String TIEBA = "tieba";//贴吧
public static final String MEDIA = "media";//报刊
public static final String VIDEO = "video";//视频
public static final String WENDA = "wenda";//问答
public static final String SHEJIAO = "shejiao";// 社交
public static final String APP = "app";
public static String WEIBO_DEFAULT_AVATOR = "http://tva1.sinaimg.cn/default/images/default_avatar_male_180.gif";
public static final String BAIDUKOUBEI = "baidukoubei";
public static final String DIANPING = "dianping";
public static final String FANGTIANXIA = "fangtianxia";
public static final String BAIDUTIEBA = "baidutieba";
public static final String SOUHU = "souhu";
public static final String XIECHENG = "xiecheng";
public static final String AUTOHOME = "autohome";
public static final String TMALL = "tmall";
public static final String YILONG = "yilong";
// 电商
public static final String ITEM = "item";
public static final String ITEM_NAME = "itemName";
public static final String ZIXUN = "zixun";
public static final String FOREIGN_NEWS = "foreign_news";
public static final String FOREIGN_MEDIA = "foreign_media";
//电商星级分数
public static final String COMMENT_SCORE = "commentScore";
// 电商数据没有星级时返回-1
public static final Double COMMENT_SCORE_NULL = -1.0;
public static final String EN_SOURCE = "enSource";
//内容识别
public static final String NORMAL = "normal";
public static final String MARKETING = "marketing";
public static final String AVATAR = "avatar";
public static final String COMMENT_ID = "commentId";
/**
* 微博信息 added by Eric 2016-10-28 15:03:31
*/
public static String WEIBO_USER_SEX = "sex";// 性别 female \ male
public static String WEIBO_USER_SEX_FEMALE = "female";// 性别 female
public static String WEIBO_USER_SEX_MALE = "male";
public static String WEIBO_USER_FRIENDS_COUNT = "friendsCount";// 关注数
public static String WEIBO_USER_FANS_COUNT = "fansCount";// 粉丝数
public static String WEIBO_ATTITUDES_COUNT = "attitudesCount";// 赞数
public static String WEIBO_COMMENTS_COUNT = "commentsCount";// 评论数
public static String WEIBO_QUOTE_COUNT = "quoteCount";// 转发数
public static String WEIBO_COUNT = "weiboCount";//用户转发微博数量
public static String WEIBO_LOCATION = "location";//用户转发微博数量
public static String WEIBO_USER_COUNTRY = "country";// 国家
public static String WEIBO_USER_PROVINCE = "province";// 省份
public static String WEIBO_USER_CITY = "city";// 城市 verified
public static String WEIBO_USER_AREA = "area";
public static String WEIBO_MUNICIPALITY_BEIJING = "北京";
public static String WEIBO_MUNICIPALITY_TIANJIN = "天津";
public static String WEIBO_MUNICIPALITY_CHONGQING = "重庆";
public static String WEIBO_MUNICIPALITY_SHANGHAI = "上海";
public static String WEIBO_USER_VERIFIED = "verified";// 认证类型
public static String WEIBO_USER_NOT_VERIFIED_VALUE = "0";// 非认证类型
public static String WEIBO_ORIGINAL_URL = "originalUrl";// 上层url
public static String WEIBO_EXPRESSION = "expression";// 表情图
public static String WEIBO_EXPRESSION_TEXT = "expressionText";// 表情文本
public static String WEIBO_FORWARD_URL = "forwardUrl";// 原文url
public static String FORWARD_USER_URL = "forwardUserUrl";// 原文url
public static String WEIBO_USER_TYPE = "userType";
public static String WEIBO_POST_SOURCE = "postSource";
public static String WEIBO_LEVEL = "level";
public static String WEIBO_REPLY_COMMENT = "replycomment";
public static String PRODUCT_ID = "productId";
/**
* 微博转发信息
*/
public static final String FORWARD_CONTENT = "forwardContent"; // 转发原文内容
public static final String FORWARD_RELEASE_DATE = "forwardReleaseDate"; // 转发原文发布时间
public static final String FORWARD_AUTHOR = "forwardAuthor"; // 转发原文作者昵称
public static final String FORWARD_QUOTE_COUNT = "forwardQuoteCount";
public static final String FORWARD_POST_SOURCE = "forwardPostSource";
public static final String FORWARD_COMMENT_COUNT = "forwardCommentsCount";
public static final String FORWARD_ATTITUDES_COUNT = "forwardAttitudesCount";
public static final String FORWARD_URL = "forwardUrl";
public static final String FORWARD_AVATAR = "forwardAvatar";
public static final String FORWARD_USER_TYPE = "forwardUserType";
public static final String FORWARD_PUBTIME = "forwardPubTime";
public static final String FORWARD_FLAG = "forwardFlag"; // 转发、回帖标志 1(转发、回帖) 0(原贴)
public static final int REGULAR_PUBTIME_QUERY = 0;
public static final int REGULAR_PUBTIME_AND_CREATETIME_QUERY = 1;
public static final List<String> TYPE_LIST = new ArrayList<>();
public static final List<Long> BACKEND_ANALYSIS_LIST = new ArrayList<>();
public static final List<Long> BACKEND_AVAILABILITY_LIST = new ArrayList<>();
public static final List<String> APP_TYPE_LIST = new ArrayList<String>();
/**
* 口碑评价对应渠道类型
*/
public static final List<String> ITEM_TYPE_LIST = new ArrayList<String>();
private static final String MEITUAN = "meituan";
private static final String DIAPING = "dianping";
public static final String ID = "id";
public static final String COMMENTS = "comments";
public static final String REPOSTS = "reposts";
public static final String ATTITUDES = "attitudes";
public static final Map<String, String> SORT_FLAG_MAPPING_MYSQL_SORT = new HashMap<>();
public static final String[] FIELD_LIST_MONITOR_NORMAL = EsQueryConditionUtils.getIncludeQueryField(ESConstant.CONTENT_SIMHASH
, ESConstant.TITLE_SIMHASH
, ESConstant.TITLE
, ESConstant.AUTHOR
, ESConstant.CONTENT
, ESConstant.SOURCE
, ESConstant.PUBTIME
, ESConstant.DOC_TYPE
, ESConstant.CHANNEL
, ESConstant.URL
, ESConstant.SYS_SENTIMENT
, ESConstant.DOC_ID
, ESConstant.DATA_ID
, ESConstant.SYS_ABSTRACT
, ESConstant.COMMENT_SCORE
, ESConstant.ATTITUDES_COUNT
, ESConstant.COMMENT_ID
, ESConstant.COMMENTS_COUNT
, ESConstant.CRAWLTIME
, ESConstant.ISDOWNLOAD
, ESConstant.FILEPATH
, ESConstant.VIDEOURL);
// ID 查询字段
public static final String[] FIELD_ID_LIST = {
ESConstant.DATA_ID,
ESConstant.DOC_ID};
public static final String[] FIELD_DATA = EsQueryConditionUtils.getIncludeQueryField(
ESConstant.AVATAR ,
ESConstant.AUTHOR,
ESConstant.DOC_ID ,
ESConstant.PUBTIME ,
ESConstant.SOURCE ,
ESConstant.URL ,
ESConstant.QUOTE_COUNT ,
ESConstant.COMMENTS_COUNT ,
ESConstant.ATTITUDES_COUNT ,
ESConstant.TITLE ,
ESConstant.CONTENT ,
ESConstant.CHANNEL ,
ESConstant.DOC_TYPE ,
ESConstant.CRAWLTIME ,
ESConstant.SYS_SENTIMENT);
// 总体分析要用的字段
public static final String[] FIELD_LIST_ANALYSIS =
EsQueryConditionUtils.getIncludeQueryField(ESConstant.PUBTIME
, ESConstant.CRAWLTIME
, ESConstant.TITLE_SIMHASH
, ESConstant.SCORE_FROM_SOURCE
, ESConstant.DOC_ID
, ESConstant.DOC_TYPE
, ESConstant.HL_KEYWORDS
, ESConstant.SOURCE
, ESConstant.SYS_SENTIMENT
, ESConstant.CONTENT_TAG
, ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_NAME
, ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_VALUE
);
// 词云查询字段
public static final String[] FIELD_CLOUD_ANALYSIS= EsQueryConditionUtils.getIncludeQueryField(
// 所有
ESConstant.HL_KEYWORDS
// 地名
, ESConstant.PLACES
// 话题
, ESConstant.HASH_TAG
// 评价
, ESConstant.OPINIONS
// 表情
, ESConstant.EXPRESSION
// ID
, ESConstant.DOC_ID
);
}

66
cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ScoreFactorEntity.java
View File

@ -1,66 +0,0 @@
package com.bfd.mf.common.util.constants;
import java.io.Serializable;
/**
* Created by Changquanyou on 2017/7/12.
* some scores of the factor
*/
public class ScoreFactorEntity implements Serializable {
private static final long serialVersionUID = 2322127803773558957L;
/**Es 默认文档评分*/
private Float tfidfScore;
/**相似文章量得分*/
private Float similarityNumber;
/**情感标签得分*/
private Double sentimentScore;
/**网站排名得分*/
private Float alexaScore;
/**渠道得分*/
private Float channelScore;
public Float getTfidfScore() {
return tfidfScore;
}
public void setTfidfScore(Float tfidfScore) {
this.tfidfScore = tfidfScore;
}
public Float getSimilarityNumber() {
return similarityNumber;
}
public void setSimilarityNumber(Float similarityNumber) {
this.similarityNumber = similarityNumber;
}
public Double getSentimentScore() {
return sentimentScore;
}
public void setSentimentScore(Double sentimentScore) {
this.sentimentScore = sentimentScore;
}
public Float getAlexaScore() {
return alexaScore;
}
public void setAlexaScore(Float alexaScore) {
this.alexaScore = alexaScore;
}
public Float getChannelScore() {
return channelScore;
}
public void setChannelScore(Float channelScore) {
this.channelScore = channelScore;
}
}

308
cl_search_api/src/main/java/com/bfd/mf/common/util/enums/CacheEnums.java
View File

@ -1,308 +0,0 @@
package com.bfd.mf.common.util.enums;
public class CacheEnums {
public enum TaskEventType {
change_doc(0, "doc变更"),
change_subject(1, "话题变更"),
change_cache_swich(2, "开启/关闭缓存开关"),
change_display_duration(3, "修改最大显示时长"),
change_white_list(4, "白名单变更"),
change_trace_back_duration(5, "修改创建之日起最大追溯时长");
private int code;
private String name;
TaskEventType(int code, String name) {
this.code = code;
this.name = name;
}
public Integer getCode() {
return code;
}
public String getName() {
return name;
}
public static TaskEventType getByCode(int code) {
for (TaskEventType taskEventType : TaskEventType.values()) {
if (taskEventType.getCode().equals(code)) {
return taskEventType;
}
}
throw new IllegalArgumentException("code error");
}
}
public enum TaskBatchType {
regular(0, "定时"),
rebuild(1, "重构"),
recover(2, "恢复");
private int code;
private String name;
TaskBatchType(int code, String name){
this.code = code;
this.name = name;
}
public Integer getCode() {
return code;
}
public String getName() {
return name;
}
public static TaskBatchType getByCode(int code) {
for (TaskBatchType taskBatchType : TaskBatchType.values()) {
if (taskBatchType.getCode().equals(code)) {
return taskBatchType;
}
}
throw new IllegalArgumentException("code error");
}
}
public enum TaskOperation {
delete(0, "删除"),
create(1, "创建"),
update(2, "修改");
private int code;
private String name;
TaskOperation(int code, String name){
this.code = code;
this.name = name;
}
public Integer getCode() {
return code;
}
public String getName() {
return name;
}
public static TaskOperation getByCode(int code) {
for (TaskOperation taskOperation : TaskOperation.values()) {
if (taskOperation.getCode().equals(code)) {
return taskOperation;
}
}
throw new IllegalArgumentException("code error");
}
}
public enum CacheSwitch {
close(0, "关闭"),
open(1, "开启"),
closeAndSearch(2, "关闭缓存且直接查询数据");
private int code;
private String name;
CacheSwitch(int code, String name){
this.code = code;
this.name = name;
}
public Integer getCode() {
return code;
}
public String getName() {
return name;
}
public static CacheSwitch getByCode(int code) {
for (CacheSwitch cacheSwitch : CacheSwitch.values()) {
if (cacheSwitch.getCode().equals(code)) {
return cacheSwitch;
}
}
throw new IllegalArgumentException("code error");
}
}
public enum CacheRecalculateStatus {
stable(0, "稳定"),
rebuild(1, "重构");
private int code;
private String name;
CacheRecalculateStatus(int code, String name){
this.code = code;
this.name = name;
}
public Integer getCode() {
return code;
}
public String getName() {
return name;
}
public static CacheRecalculateStatus getByCode(int code) {
for (CacheRecalculateStatus cacheRecalculateStatus : CacheRecalculateStatus.values()) {
if (cacheRecalculateStatus.getCode().equals(code)) {
return cacheRecalculateStatus;
}
}
throw new IllegalArgumentException("code error");
}
}
public enum CacheRefreshStatus {
stable(0, "稳定"),
rebuild(1, "重构");
private int code;
private String name;
CacheRefreshStatus(int code, String name){
this.code = code;
this.name = name;
}
public Integer getCode() {
return code;
}
public String getName() {
return name;
}
public static CacheRefreshStatus getByCode(int code) {
for (CacheRefreshStatus cacheRefreshStatus : CacheRefreshStatus.values()) {
if (cacheRefreshStatus.getCode().equals(code)) {
return cacheRefreshStatus;
}
}
throw new IllegalArgumentException("code error");
}
}
public enum UpdateSubject {
no(0, "不更新"),
yes(1, "更新");
private int code;
private String name;
UpdateSubject(int code, String name){
this.code = code;
this.name = name;
}
public Integer getCode() {
return code;
}
public String getName() {
return name;
}
public static UpdateSubject getByCode(int code) {
for (UpdateSubject updateSubject : UpdateSubject.values()) {
if (updateSubject.getCode().equals(code)) {
return updateSubject;
}
}
throw new IllegalArgumentException("code error");
}
}
public static final Integer UpdateSubjectNumLeft = 60;
public enum TaskSubjectStatus{
kill(-2, "杀死"),
fail(-1, "失败"),
processing(0, "进行中"),
success(1, "成功"),
recover(2, "已恢复");
private int code;
private String name;
TaskSubjectStatus(int code, String name){
this.code = code;
this.name = name;
}
public Integer getCode() {
return code;
}
public String getName() {
return name;
}
public static TaskSubjectStatus getByCode(int code) {
for (TaskSubjectStatus taskSubjectStatus : TaskSubjectStatus.values()) {
if (taskSubjectStatus.getCode().equals(code)) {
return taskSubjectStatus;
}
}
throw new IllegalArgumentException("code error");
}
}
public enum TaskHandle {
no(0, "不处理"),
yes(1, "需处理");
private int code;
private String name;
TaskHandle(int code, String name){
this.code = code;
this.name = name;
}
public Integer getCode() {
return code;
}
public String getName() {
return name;
}
public static TaskHandle getByCode(int code) {
for (TaskHandle taskHandle : TaskHandle.values()) {
if (taskHandle.getCode().equals(code)) {
return taskHandle;
}
}
throw new IllegalArgumentException("code error");
}
}
public enum TaskBatchStage{
undo(-1, "未执行"),
start(0, "开始执行"),
childrenCreate(1, "开始创建子任务"),
childrenFinish(2, "创建子任务完毕");
private int code;
private String name;
TaskBatchStage(int code, String name){
this.code = code;
this.name = name;
}
public Integer getCode() {
return code;
}
public String getName() {
return name;
}
public static TaskBatchStage getByCode(int code) {
for (TaskBatchStage taskBatchStage : TaskBatchStage.values()) {
if (taskBatchStage.getCode().equals(code)) {
return taskBatchStage;
}
}
throw new IllegalArgumentException("code error");
}
}
}

78
cl_search_api/src/main/java/com/bfd/mf/common/util/enums/DataExportStatus.java
View File

@ -1,78 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.util.enums;
public enum DataExportStatus {
INIT(0, "Initial", "等待处理"),
PROCESSING(1, "Processing", "处理中"),
FAIL(2, "Failed", "处理失败"),
SUCCESS(3, "Success", "完成"),
NODATA(4, "NoData", "暂无数据"),
UNKNOWN(-1, "Unknown", "未知错误");
private int code;
private String desc, locale;
DataExportStatus(int code, String desc, String locale) {
this.code = code;
this.desc = desc;
this.locale = locale;
}
public int getCode() {
return code;
}
public void setCode(int code) {
this.code = code;
}
public String getDesc() {
return desc;
}
public void setDesc(String desc) {
this.desc = desc;
}
public String getLocale() {
return locale;
}
public void setLocale(String locale) {
this.locale = locale;
}
public static DataExportStatus getStatusByCode(int code) {
switch (code) {
case 0:
return INIT;
case 1:
return PROCESSING;
case 2:
return FAIL;
case 3:
return SUCCESS;
case 4:
return NODATA;
default:
return UNKNOWN;
}
}
}

119
cl_search_api/src/main/java/com/bfd/mf/common/util/enums/RTCodeEnum.java
View File

@ -1,119 +0,0 @@
package com.bfd.mf.common.util.enums;
import com.alibaba.fastjson.JSONObject;
public enum RTCodeEnum {
C_OK(0, "OK"), //
C_DONE(1, "Done"), //
C_FAIL(-1, "Failed"),
/**
* ES Index
*/
C_INDEX_EXISTS(-3, "Index Exists"),
// 自定义状态码
C_TOPIC_DISABLED(-2, "话题渠道为空,请运营人员确注意操作！！！"),
// Param Issue: 3**
C_PARAM_ERROR(300, "Input Param Error"),
C_TOKEN_ERROR(301, "Token Validation Error"),
C_CAPTCHA_ERROR(302, "验证码错误，请重试"),
C_APPLY_DUPLICATION(303, "申请试用信息已存在, 请不要重复提交"),
C_UNBIND(304, "微信账号未绑定"),
C_STATE_EXIST(305, "请勿重复请求"),
// Data Issue: 4**
C_NO_DATA_FOUND(400, "No Data Found"),
C_DATA_ERROR_PAGETIME_EXPIRE(401, "页面超时不可用，请刷新重试"),
// System Service Issue: 5**
C_SERVICE_NOT_AVAILABLE(500, "系统服务不可用，请联系管理员"),
C_BIZ_IMPACT_DAYS_NOT_SUPPRTTED(601, "Impact days not supported, we only support for 1,5,10,20 for free"),
C_SUBJECT_GRAMMAR_NOT_SUPPRTTED(701, "语法错误,请修改"),
C_SUBJECT_GRAMMAR_EXPERTKEY_WORD(913, "关键词不允许超过50个"),
// Eric added @2016年11月02日15:27:10
C_SUBJECT_DEFINITION_NOT_ACCURATE(702, "请不要设置过于宽泛的关键词组合, 这样会采集过多不相关的文章"),
C_SUBJECT_ZERO_GRAMMAR_NOT_SUPPRTTED(703, "该语法查询不到内容,请精确查询意图"),
C_SUBJECT_MORE_GRAMMAR_NOT_SUPPRTTED(704, "语法配置过于宽泛,请精确查询意图"),
C_PERMISSION_ERROR_USER_RANOUT(801, "用户数量达到最大限制, 请联系管理员升级权限"),
C_PERMISSION_ERROR_TOPIC_RANOUT(802, "话题数量超过限制, 请联系客服人员处理"),
C_PERMISSION_ERROR_LOGIN_FAIL(803, "用户名或密码不正确，请重新登录"),
C_PERMISSION_ERROR_COMPANY_EXPIRE(804, "客户账号已过期或不可用，请联系管理员"),
C_PERMISSION_ERROR_SENDTIME_EXPIRE(805, "发送时间超过限制不可用，请修改发送时间"),
C_PERMISSION_ERROR_SCREEN_RANOUT(806, "屏蔽数量达到最大限制, 请联系管理员升级权限"),
C_PERMISSION_ERROR_CODE_EXPIRE(807, "code已过期或不可用"),
C_PERMISSION_ERROR_SHOPURL_RANOUT(808, "用户url数量达到最大限制, 请联系管理员升级权限"),
C_PERMISSION_ERROR_COMMENT_CLOSED(809, "客户账号口碑功能未开启，请联系管理员"),
C_COMPANY_DEADLINE(909, "客户过期!"),
C_UPLOAD_IMG_FAIL(901, "上传失败"),
C_VALID_PASSWORD_ERROR(904, "原始密码错误"),
C_USER_PERMSSION_ERROR(908, "当前账号无任何权限，请联系管理员"),
C_USER_STATUS_ERROR(910, "用户被禁用，请联系管理员"),
C_KEYWORD_LIMIT(902, "关键词不能大于客户配置词数！"),
C_EXCLUDE_LIMIT(903, "排除词不能大于客户配置词数！"),
C_LACK_SPREADNUM(904, "剩余转发数不足，请与工作人员联系!"),
C_WEIBO_CRAWL_ERR(905, "加载失败，请检查该微博是否被删除!"),
C_WEIBO_URL_ERR(906, "微博链接不符合规范!"),
C_FAVOUR_NUMBER_ERR(907, "数量超过阈值限制"),
C_WEIBO_TOTAL_COUNT(908, "本帖转发和评论量超过100000条，请更换其他转评量和评论量小于100000的帖子。"),
C_ACCOUNT_NICK_NAME(911, "该账号已存在，不能重复添加"),
C_ACCOUNT_NO_NICK_NAME(912, "该用户昵称不存在");
private int code;
private String desc;
RTCodeEnum(int code, String desc) {
this.code = code;
this.desc = desc;
}
public JSONObject toJSON() {
JSONObject jsonObject = new JSONObject();
jsonObject.put("code", code);
jsonObject.put("desc", desc);
return jsonObject;
}
public int getCode() {
return code;
}
public void setCode(int code) {
this.code = code;
}
public String getDesc() {
return desc;
}
public void setDesc(String desc) {
this.desc = desc;
}
}

99
cl_search_api/src/main/java/com/bfd/mf/common/util/enums/SpreadEnums.java
View File

@ -1,99 +0,0 @@
package com.bfd.mf.common.util.enums;
import java.util.HashSet;
import java.util.Set;
/**
* Created by Shang Gaohui on 2017/5/31.
*/
public class SpreadEnums {
public static class TREND {
public static final Integer HOUR_LIMIT = 48;
public static final Integer UNIT_HOUR = 0;
public static final Integer UNIT_DAY = 1;
public static final Integer PERIOD_NUM = 10;
}
public static class LIST {
public static final Integer PAGE_DEFAULT_NUM = 20;
public static final String KEY_PREFIX = "spread_list_";
public static final Long EXPIRE_TIME = 24 * 60 * 60 * 1000L;
public static final String SPREAD_FIELD_PUBTIME = "pubTime";
public static final String SPREAD_FIELD_CRAWLTIME = "crawlTime";
public static final String SPREAD_FIELD_DOCID = "docId";
}
public enum LIST_TYPE {
similar(0, "相似"),
accurate(1, "精确");
private Integer code;
private String value;
LIST_TYPE(Integer code, String value) {
this.code = code;
this.value = value;
}
public String getValue() {
return value;
}
public Integer getCode() {
return code;
}
}
public enum SEARCH_TYPE {
title(0, "标题"),
source(1, "来源"),
originalSource(2, "转载出处");
private Integer code;
private String value;
SEARCH_TYPE(Integer code, String value) {
this.code = code;
this.value = value;
}
public String getValue() {
return value;
}
public Integer getCode() {
return code;
}
}
public enum SORT_TYPE {
PUBTIME_DESC_SORT_FLAG("timeDesc"),
PUBTIME_ASC_SORT_FLAG("timeAsc");
private String value;
SORT_TYPE(String value) {
this.value = value;
}
public String getValue() {
return value;
}
}
public static class OriginalSource {
public static final Set<String> illegal = new HashSet() {{
add("null");
add("未知");
add("其他");
add("其它");
add("其它的");
add("互联网");
add("来源于网络");
add("来源于互联网");
add("无");
}};
}
}

32
cl_search_api/src/main/java/com/bfd/mf/common/util/es/MonitorConstant.java
View File

@ -1,32 +0,0 @@
package com.bfd.mf.common.util.es;
import com.bfd.mf.common.util.constants.ESConstant;
import java.util.HashMap;
import java.util.Map;
public class MonitorConstant {
/**
* 报告数目
*/
public static final int SHOW_REPORT_NUM = 21;
/***
* 情感中英文对照
*/
public static Map<String, String> emotionEngMaps() {
Map<String, String> maps = new HashMap<>();
maps.put(ESConstant.NEGATIVE_TAG, ESConstant.NEGATIVE_CN_TAG);
maps.put(ESConstant.COMMON_TAG, ESConstant.COMMON_CN_TAG);
return maps;
}
public static Map<String, String> emotionEngByThresholdMaps() {
Map<String, String> maps = new HashMap<>();
maps.put(ESConstant.SENTIMENTAL_MIN + "-" + ESConstant.SENTIMENTAL_THRESHOLD, ESConstant.NEGATIVE_CN_TAG);
maps.put(ESConstant.SENTIMENTAL_MID + "-" + ESConstant.SENTIMENTAL_MAX, ESConstant.COMMON_CN_TAG);
maps.put(ESConstant.SENTIMENTAL_THRESHOLD + "-" + ESConstant.SENTIMENTAL_MID, ESConstant.NEUTER_CN_TAG);
return maps;
}
}

52
cl_search_api/src/main/java/com/bfd/mf/common/util/es/MonitorUtils.java
View File

@ -1,52 +0,0 @@
package com.bfd.mf.common.util.es;
import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.mf.common.web.vo.view.monitor.ESMonitorBaseEntity;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class MonitorUtils {
private static final String SPLIT = "_";
private static Logger logger = LoggerFactory.getLogger(MonitorUtils.class);
public static ESMonitorBaseEntity buildESMonitorListEntity(String dataId) {
ESMonitorBaseEntity esMonitorBaseEntity = new ESMonitorBaseEntity();
esMonitorBaseEntity.setDataId(dataId);
return esMonitorBaseEntity;
}
public static void parseTimeSeries(
SearchResponse scrollResponse,
Map<String, Integer> docIdWithAreaMap,
Map<String, ESMonitorBaseEntity> esMonitorEntityMap) {
if (null != scrollResponse) {
SearchHits hits = scrollResponse.getHits();
SearchHit[] searchHists = hits.getHits();
for (SearchHit searchHit : searchHists) {
if (null != searchHit.getSourceAsMap().get(ESConstant.DATA_ID)) {
String dataId = searchHit.getSourceAsMap().get(ESConstant.DATA_ID).toString();
if(docIdWithAreaMap.containsKey(dataId)){
docIdWithAreaMap.put(dataId, 2);
}else {
docIdWithAreaMap.put(dataId, 1);
}
ESMonitorBaseEntity esMonitorBaseEntity = MonitorUtils.buildESMonitorListEntity(dataId);
esMonitorEntityMap.put(dataId, esMonitorBaseEntity);
}else{
System.out.println("[MonitorUtils] ??? : "+searchHit.getSourceAsMap().toString());
}
}
}
}
}

177
cl_search_api/src/main/java/com/bfd/mf/common/util/slice/SliceScrollUtil.java
View File

@ -1,177 +0,0 @@
package com.bfd.mf.common.util.slice;
import com.bfd.mf.common.service.es.ClusterService;
import com.bfd.mf.common.thread.SubjectQueryDataService;
import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.mf.common.web.entity.mysql.cache.Cluster;
import com.bfd.mf.common.web.vo.params.QueryRequest;
import com.bfd.mf.common.web.vo.view.monitor.ESMonitorEntity;
import com.bfd.mf.config.BFDApiConfig;
import com.bfd.mf.service.SearchDataService;
import com.bfd.nlp.common.util.object.TObjectUtils;
import com.bfd.nlp.common.util.string.TStringUtils;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.*;
import java.util.stream.Collectors;
@Component
public class SliceScrollUtil {
private static final Logger logger = LoggerFactory.getLogger(SliceScrollUtil.class);
@Autowired
private ClusterService clusterService;
@Autowired
private SubjectQueryDataService subjectQueryDataService;
private Object existHighEm(String str) {
if (TStringUtils.isNotEmpty(str)) {
if (str.contains("<em>") || str.contains("</em>")) {
return str;
}
}
return "";
}
/**
* 解析数据
*/
private void parseEsData(SearchResponse searchResponse, List<ESMonitorEntity> list) {
if (searchResponse != null) {
logger.info("[SliceScrollUtil] parseEsData : ");
SearchHits hits = searchResponse.getHits();
SearchHit[] searchHists = hits.getHits();
for (SearchHit searchHit : searchHists) {
ESMonitorEntity esMonitorBaseEntity = new ESMonitorEntity();
Map<String, Object> sourceAsMap = searchHit.getSourceAsMap();
esMonitorBaseEntity.setDocId((String) sourceAsMap.get(ESConstant.DOC_ID));
if(sourceAsMap.containsKey(ESConstant.DOC_TYPE)) {
esMonitorBaseEntity.setDocType((String) sourceAsMap.get(ESConstant.DOC_TYPE));
}
if(sourceAsMap.containsKey(ESConstant.PUBTIME)) {
esMonitorBaseEntity.setPubTime(Long.valueOf(sourceAsMap.get(ESConstant.PUBTIME).toString()));
}
if(sourceAsMap.containsKey(ESConstant.SOURCE)) {
esMonitorBaseEntity.setSource((String) sourceAsMap.get(ESConstant.SOURCE));
}
if(sourceAsMap.containsKey(ESConstant.URL)) {
esMonitorBaseEntity.setUrl((String) sourceAsMap.get(ESConstant.URL));
}
if(sourceAsMap.containsKey(ESConstant.URL_HASH)) {
esMonitorBaseEntity.setUrlHash((String) sourceAsMap.get(ESConstant.URL_HASH));
}
if(sourceAsMap.containsKey(ESConstant.AVATAR)){
esMonitorBaseEntity.setAvatar(String.valueOf(sourceAsMap.getOrDefault(ESConstant.AVATAR, "")));
}
if(sourceAsMap.containsKey(ESConstant.CRAWLTIME)) {
esMonitorBaseEntity.setCrawlTime(Long.valueOf(sourceAsMap.get(ESConstant.CRAWLTIME).toString()));
if ("null".equalsIgnoreCase(String.valueOf(sourceAsMap.get(ESConstant.CRAWLTIME)))) {
esMonitorBaseEntity.setCrawlTime(Long.valueOf(sourceAsMap.get(ESConstant.PUBTIME).toString()));
}
}
if(sourceAsMap.containsKey(ESConstant.AUTHOR)) {
esMonitorBaseEntity.setAuthor(String.valueOf(sourceAsMap.get(ESConstant.AUTHOR)));
}
if(sourceAsMap.containsKey(ESConstant.POST_SOURCE)) {
esMonitorBaseEntity.setPostSource(Optional.ofNullable((String) sourceAsMap.get(ESConstant.POST_SOURCE)).orElse(""));
}
if(sourceAsMap.containsKey(ESConstant.ATTITUDES_COUNT)) {
esMonitorBaseEntity.setAttitudesCount(sourceAsMap.get(ESConstant.ATTITUDES_COUNT).toString());
}
if(sourceAsMap.containsKey(ESConstant.QUOTE_COUNT)) {
esMonitorBaseEntity.setQuoteCount(getCountForEsParams(sourceAsMap.get(ESConstant.QUOTE_COUNT), sourceAsMap.get(ESConstant.DOC_ID)));
}
if(sourceAsMap.containsKey(ESConstant.SYS_SENTIMENT)) {
esMonitorBaseEntity.setSysSentiment(Double.valueOf(sourceAsMap.get(ESConstant.SYS_SENTIMENT).toString()));
}
if(sourceAsMap.containsKey(ESConstant.COMMENT_ID)) {
esMonitorBaseEntity.setCommentId((String) sourceAsMap.getOrDefault(ESConstant.COMMENT_ID, ""));
}
if(sourceAsMap.containsKey(ESConstant.AVAILABILITY)) {
esMonitorBaseEntity.setAvailability((Integer) sourceAsMap.get(ESConstant.AVAILABILITY));
}
if(sourceAsMap.containsKey(ESConstant.PICTURE_LIST)) {
esMonitorBaseEntity.setPictureList((List) sourceAsMap.getOrDefault(ESConstant.PICTURE_LIST, new ArrayList<>()));
}
try {
esMonitorBaseEntity.setHlKeyWords((List<String>) sourceAsMap.getOrDefault(ESConstant.HL_KEYWORDS, new ArrayList<>()));
} catch (Exception e) {
esMonitorBaseEntity.setHlKeyWords(new ArrayList<>());
}
esMonitorBaseEntity.setPlaces((List<String>) sourceAsMap.getOrDefault(ESConstant.PLACES, new ArrayList<>()));
esMonitorBaseEntity.setExpression((List) sourceAsMap.getOrDefault(ESConstant.EXPRESSION, new ArrayList<>()));
esMonitorBaseEntity.setHashTag((List<String>) sourceAsMap.getOrDefault(ESConstant.HASH_TAG, new ArrayList<>()));
esMonitorBaseEntity.setOpinions((List<String>) sourceAsMap.getOrDefault(ESConstant.OPINIONS, new ArrayList<>()));
list.add(esMonitorBaseEntity);
}
}
}
public Integer getCountForEsParams(Object o1, Object o2) {
if (TObjectUtils.isNull(o1) || o1.equals("")) {
if (ESConstant.WEI_BO.equals(o2.toString()) || ESConstant.ITEM.equals(o2.toString())) {
return 0;
}
return 0;
} else {
return Integer.valueOf(o1.toString().replaceAll(" ", ""));
}
}
public List<ESMonitorEntity> fetchResultSubjectCache(QueryRequest queryRequest,String[] fieldList) {
List<ESMonitorEntity> jsonObjectList = new ArrayList<>();
long calculateStartTime = System.currentTimeMillis();
Cluster cluster = null;
List<String> currentIndexList = null;
if(null != queryRequest.getSubjectId()){
System.out.println("subjectId = " + queryRequest.getSubjectId());
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.mini_cluster_type); // 111
currentIndexList = subjectQueryDataService.getIndexBySubjectIds(cluster, queryRequest.getSubjectId());
}else{
logger.info("[SearchKeywordsCouldService] dataAnalysisCloud : 查询 全局数据");
cluster = clusterService.findClusterByType(Cluster.CLUSTER_TYPE.normal_cluster_type); // 109
currentIndexList = subjectQueryDataService.getIndexListByTimeRange(cluster, queryRequest.getStartTime(),queryRequest.getEndTime());
}
Long clusterId = cluster.getId();
logger.info("[SearchKeywordsCouldService] dataAnalysisCloud : queryDataList clusterId = " + clusterId + " ; currentIndexList :" + currentIndexList.toString());
logger.info("==========进入数据分析Es and Cache,计算开始执行============");
try {
List<SearchResponse> responseList = new ArrayList<>();
List<SearchResponse> searchResponseEsList =subjectQueryDataService.fetchResponseDataFromCache(
queryRequest,
clusterId,
currentIndexList,
fieldList);
if (Optional.ofNullable(searchResponseEsList).isPresent()) {
responseList.addAll(searchResponseEsList);
parseResponseList(jsonObjectList, responseList);
}
logger.info("==========进入数据分析Es and Cache,计算结束执行,消耗时长为：{}============", System.currentTimeMillis() - calculateStartTime);
} catch (Exception e) {
e.printStackTrace();
}
return jsonObjectList.stream().distinct().collect(Collectors.toList());
}
public void parseResponseList(List<ESMonitorEntity> jsonObjectList, List<SearchResponse> responseList) {
for (SearchResponse searchResponse : responseList) {
// 解析数据
parseEsData(searchResponse, jsonObjectList);
}
}
}

334
cl_search_api/src/main/java/com/bfd/mf/common/util/spread/SpreadServiceUtil.java
View File

@ -1,334 +0,0 @@
package com.bfd.mf.common.util.spread;
import com.alibaba.fastjson.JSONObject;
import com.bfd.mf.common.util.enums.SpreadEnums;
import com.bfd.mf.common.util.utility.DateUtil;
import com.bfd.nlp.common.util.object.TObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class SpreadServiceUtil {
private static final Logger logger = LoggerFactory.getLogger(SpreadServiceUtil.class);
private static final Long ONE_DAY = 24 * 60 * 60 * 1000L;
private static final Long ONE_HOUR = 60 * 60 * 1000L;
// static {
// try {
// while (!ApplicationUtil.initialFinsih) {
// logger.warn("[SpreadServiceUtil Initial] AbsSpringServiceHolder: Wait spring content initialization, sleep 1s...");
// Thread.sleep(1000);
// }
//
// oemUrlCommonService = ApplicationUtil.getApplicationContext().getBean(OemUrlCommonService.class);
// bfdApiConfig = ApplicationUtil.getApplicationContext().getBean(BFDApiConfig.class);
//
// if (TObjectUtils.isNull(oemUrlCommonService)) {
// throw new Exception("oemUrlCommonService is null");
// }
//
// } catch (Exception e) {
// logger.error("[SpreadServiceUtil Initial] Object {} is NULL", e.getMessage(), e);
// System.exit(-1);
// }
// }
/**
* 获取三个月前的起始时间
*/
public static Long getTimeOfThreeMonthsAgo(Long time) {
if (TObjectUtils.isNull(time)) {
time = System.currentTimeMillis();
}
Date endDate = new Date(time);
Calendar calendar = Calendar.getInstance();
calendar.setTime(endDate);
calendar.add(Calendar.MONTH, -3);
calendar.set(Calendar.DAY_OF_MONTH, 1);
calendar.set(Calendar.HOUR_OF_DAY, 0);
calendar.set(Calendar.MINUTE, 0);
calendar.set(Calendar.SECOND, 0);
calendar.set(Calendar.MILLISECOND, 0);
logger.info("[SpreadServiceUtil] getTimeOfThreeMonthsAgo: result is {}", calendar.getTime());
return calendar.getTimeInMillis();
}
private static Long getEndTime(Long time, Integer type) {
Long startTime;
if (type.equals(SpreadEnums.TREND.UNIT_HOUR)) {
startTime = getEndTimeOfHour(time);
} else {
startTime = getEndTimeOfDay(time);
}
return startTime;
}
private static Long getEndTimeOfDay(Long time) {
Calendar calendar = Calendar.getInstance();
calendar.setTimeInMillis(time);
calendar.set(Calendar.HOUR_OF_DAY, 23);
calendar.set(Calendar.MINUTE, 59);
calendar.set(Calendar.SECOND, 59);
calendar.set(Calendar.MILLISECOND, 999);
return calendar.getTimeInMillis();
}
private static Long getEndTimeOfHour(Long time) {
Calendar calendar = Calendar.getInstance();
calendar.setTimeInMillis(time);
calendar.set(Calendar.MINUTE, 59);
calendar.set(Calendar.SECOND, 59);
calendar.set(Calendar.MILLISECOND, 999);
return calendar.getTimeInMillis();
}
public static Long[] getTimeList1(Long startTime, Long endTime, Integer type) {
Long[] timeList = null;
try {
Long interval;
if (type.equals(SpreadEnums.TREND.UNIT_HOUR)) {
interval = ONE_HOUR;
} else if (type.equals(SpreadEnums.TREND.UNIT_DAY)) {
interval = ONE_DAY;
} else {
throw new Exception(" !!! type is illegal !!!");
}
Double intervalNum = Math.ceil((endTime - startTime) / (double) interval);
timeList = new Long[intervalNum.intValue() + 1];
Long time = startTime;
for (int i = 0; i < intervalNum.intValue(); i++) {
timeList[i] = time;
time += interval;
}
timeList[intervalNum.intValue()] = endTime;
logger.info("[SpreadServiceUtil] getTimeList: startTime is {}, endTime is {}, timeList is {}", startTime, endTime, timeList);
} catch (Exception e) {
logger.error("[SpreadServiceUtil] getTimeList: failed, startTime is {}, endTime is {}, timeList is {}, error is ", startTime, endTime, timeList, e);
}
return timeList;
}
public static Long[] getTimeList2(Long startTime, Long endTime, Integer type) {
Long[] timeList = null;
try {
Long interval;
if (type.equals(SpreadEnums.TREND.UNIT_HOUR)) {
interval = ONE_HOUR;
} else if (type.equals(SpreadEnums.TREND.UNIT_DAY)) {
interval = ONE_DAY;
} else {
throw new Exception(" !!! type is illegal !!!");
}
List<Long> timeListTemp = new ArrayList<>();
timeListTemp.add(startTime);
if (startTime.equals(endTime)) {
timeList = new Long[timeListTemp.size()];
return timeListTemp.toArray(timeList);
}
Long endTimeOfStart = getEndTime(endTime, type);
if (endTimeOfStart >= endTime) {
timeListTemp.add(endTime);
timeList = new Long[timeListTemp.size()];
return timeListTemp.toArray(timeList);
}
for (long time = endTimeOfStart + interval; time < endTime; time += interval) {
timeListTemp.add(time);
}
timeListTemp.add(endTime);
timeList = new Long[timeListTemp.size()];
timeListTemp.toArray(timeList);
logger.info("[SpreadServiceUtil] getTimeList: startTime is {}, endTime is {}, timeList is {}", startTime, endTime, timeList);
} catch (Exception e) {
logger.error("[SpreadServiceUtil] getTimeList: failed, startTime is {}, endTime is {}, timeList is {}, error is ", startTime, endTime, timeList, e);
}
return timeList;
}
/**
* 将Long型时间序列列表转换为String型列表
*/
public static String[] convertToTimeStringList(Long[] timeList, Integer type) {
String[] timeStringList = new String[timeList.length];
try {
String dateFormat;
if (Objects.equals(type, SpreadEnums.TREND.UNIT_HOUR)) {
dateFormat = DateUtil.TIME_FORMAT;
} else if (Objects.equals(type, SpreadEnums.TREND.UNIT_DAY)) {
dateFormat = DateUtil.DATE_FORMAT;
} else {
throw new Exception(" !!! type is illegal !!!");
}
for (int i = 0; i < timeList.length; i++) {
timeStringList[i] = DateUtil.parseDateByFormat(timeList[i], dateFormat);
}
} catch (Exception e) {
logger.error("[SpreadServiceUtil] getTimeStringList: failed, timeList is {}, error is ", Arrays.asList(timeStringList), e);
}
return timeStringList;
}
/**
* 构造list搜索返回值
*/
public static JSONObject getSearchListData(List<JSONObject> spreadList, Integer totalNum, Integer limit) {
JSONObject data = new JSONObject();
Double pageNum = 0d;
try {
pageNum = Math.ceil((double) totalNum / limit);
} catch (Exception e) {
logger.error("[SpreadServiceUtil] getSearchListData: failed, spreadList is {}, limit is {}, error is ",
spreadList, limit, e);
}
data.put("totalNum", totalNum);
data.put("pageNum", pageNum.intValue());
data.put("spreadList", spreadList);
return data;
}
/**
* 精确过滤
*/
public static Boolean accurateSearchResult(JSONObject spread, String field, String text) {
if (spread.containsKey(field) && spread.getString(field).equals(text)) {
return true;
}
return false;
}
/**
* 关键词过滤
*/
public static Boolean searchResult(JSONObject spread, String field, String text) {
if (spread.containsKey(field) && spread.getString(field).contains(text)) {
return true;
}
return false;
}
/**
* 按照指定排序规则排序
*/
public static List<JSONObject> sortBySortFlag(List<JSONObject> spreadList, String sortFlag) {
logger.info("[SpreadServiceUtil] sortBySortFlag: -排序前-spreadList is {}, sortFlag is {}", spreadList, sortFlag);
try {
if (sortFlag.equals(SpreadEnums.SORT_TYPE.PUBTIME_DESC_SORT_FLAG.getValue())) {
// 按时间倒序
Collections.sort(spreadList, new Comparator<JSONObject>() {
@Override
public int compare(JSONObject o1, JSONObject o2) {
// 发布时间相同的，则再看采集时间
return o2.getLong(SpreadEnums.LIST.SPREAD_FIELD_PUBTIME).compareTo(o1.getLong(SpreadEnums.LIST.SPREAD_FIELD_PUBTIME)) == 0 ?
o2.getLong(SpreadEnums.LIST.SPREAD_FIELD_CRAWLTIME).compareTo(o1.getLong(SpreadEnums.LIST.SPREAD_FIELD_CRAWLTIME)) :
o2.getLong(SpreadEnums.LIST.SPREAD_FIELD_PUBTIME).compareTo(o1.getLong(SpreadEnums.LIST.SPREAD_FIELD_PUBTIME));
}
});
}
if (sortFlag.equals(SpreadEnums.SORT_TYPE.PUBTIME_ASC_SORT_FLAG.getValue())) {
// 按时间正序
Collections.sort(spreadList, new Comparator<JSONObject>() {
@Override
public int compare(JSONObject o1, JSONObject o2) {
// 发布时间相同的，则再看采集时间
return o1.getLong(SpreadEnums.LIST.SPREAD_FIELD_PUBTIME).compareTo(o2.getLong(SpreadEnums.LIST.SPREAD_FIELD_PUBTIME)) == 0 ?
o1.getLong(SpreadEnums.LIST.SPREAD_FIELD_CRAWLTIME).compareTo(o2.getLong(SpreadEnums.LIST.SPREAD_FIELD_CRAWLTIME)) :
o1.getLong(SpreadEnums.LIST.SPREAD_FIELD_PUBTIME).compareTo(o2.getLong(SpreadEnums.LIST.SPREAD_FIELD_PUBTIME));
}
});
}
logger.info("[SpreadServiceUtil] sortBySortFlag: -排序后-spreadList is {}, sortFlag is {}", spreadList, sortFlag);
} catch (Exception e) {
logger.error("[SpreadServiceUtil] sortBySortFlag: failed, spreadList is {}, error is ", spreadList, e);
}
return spreadList;
}
/**
* 获取指定页面list
*/
public static List<JSONObject> getPage(List<JSONObject> spreadList, Integer start, Integer limit) {
List<JSONObject> pageList = new ArrayList<>();
try {
if (start >= spreadList.size()) {
return pageList;
}
int end = spreadList.size() >= start + limit ? start + limit : spreadList.size();
pageList = spreadList.subList(start, end);
} catch (Exception e) {
logger.error("[SpreadServiceUtil] getPage: failed, spreadList is {}, " +
"start is {}, limit is {}, error is ", spreadList, start, limit, e);
}
return pageList;
}
/**
* 去除括号及括号内的内容
*/
private static String removeBrackets(String value) {
logger.info("[SpreadServiceUtil] removeBrackets: value is {}", value);
try {
// 中文括号修正为英文括号
value = value.trim().replaceAll("（", "(").replaceAll("）", ")");
Pattern pattern = Pattern.compile("\\((.*?)\\)"); //英文括号
Matcher matcher = pattern.matcher(value);
while (matcher.find()) {
value = value.replaceAll(matcher.group(0), ""); // 0是包括括号, 1是只取内容
logger.info("[SpreadServiceUtil] removeBrackets: value is {}, remove string is {}", value, matcher.group(1));
}
value = value.trim().replaceAll("\\(", "").replaceAll("\\)", "");
logger.info("[SpreadServiceUtil] removeBrackets: result value is {}", value);
} catch (Exception e) {
logger.error("[SpreadServiceUtil] removeBrackets: failed, value is {}, error is ", value, e);
}
return value;
}
/**
* 过滤所有以"<"开头以">"结尾的标签
*/
private static String filterHtml(String value) {
logger.info("[SpreadServiceUtil] filterHtml: value is {}", value);
try {
String regxpForHtml = "<([^>]*)>"; // 过滤所有以<开头以>结尾的标签
Pattern pattern = Pattern.compile(regxpForHtml);
Matcher matcher = pattern.matcher(value);
StringBuffer sb = new StringBuffer();
while (matcher.find()) {
matcher.appendReplacement(sb, "");
}
matcher.appendTail(sb);
logger.info("[SpreadServiceUtil] filterHtml: result value is {}", sb.toString());
return sb.toString();
} catch (Exception e) {
logger.error("[SpreadServiceUtil] filterHtml: failed, value is {}, error is ", value, e);
}
return value;
}
/**
* 过滤特殊字符
*/
public static String filter(String value) {
// 合法性过滤
if (StringUtils.isEmpty(value) || SpreadEnums.OriginalSource.illegal.contains(value.toLowerCase())) {
return "";
}
// 过滤括号
String valueAfterRemoveBrackets = removeBrackets(value);
// 过滤html标签
String valueAfterFilterHtml = filterHtml(valueAfterRemoveBrackets);
return valueAfterFilterHtml;
}
}

1042
cl_search_api/src/main/java/com/bfd/mf/common/util/subject/ExpNodeUtil.java
File diff suppressed because it is too large
View File

911
cl_search_api/src/main/java/com/bfd/mf/common/util/subject/ExpressionParser.java
View File

@ -1,911 +0,0 @@
package com.bfd.mf.common.util.subject;
import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.mf.common.util.OperatorUtil;
import com.bfd.mf.common.web.vo.params.expression.ExpObject;
import com.bfd.mf.common.web.vo.params.expression.HalfQuartChar;
import org.elasticsearch.common.Strings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
public class ExpressionParser {
private static final Logger logger = LoggerFactory.getLogger(ExpressionParser.class);
/**
* 表达式预处理
*/
public static Vector<Object> preTreatExpression(String exp, String[] fields) {
if (Strings.isNullOrEmpty(exp)) {
return null;
}
exp = exp.trim();
Vector<Object> v = new Vector<Object>();
Stack<HalfQuartChar> left = new Stack<>();// '<'
Stack<HalfQuartChar> leftQ = new Stack<>();// '('
Stack<HalfQuartChar> leftS = new Stack<>();// “ 或"
int begin = 0, end = 0;
while (end < exp.length()) {
char c = exp.charAt(end);
// 处理<>内部空格====begin
if (c == '<' || c == '《') {
left.push(new HalfQuartChar(c, end));
} else if (c == '》' || c == '>') {
if (!left.isEmpty()) {
left.pop();
}
} else if (c == '(' || c == '（') {
leftQ.push(new HalfQuartChar(c, end));
} else if (c == ')' || c == '）') {
if (!leftQ.isEmpty()) {
leftQ.pop();
}
} else if (c == '"' || c == '“') {
leftS.push(new HalfQuartChar(c, end));
} else if (c == '"' || c == '”') {
if (!leftS.isEmpty()) {
leftS.pop();
}
}
// 可能是<(A) (B)>,(A)和(B)之间的空格
if (c == ' ' && leftS.isEmpty()) {// 没有在“”内部
if (!left.isEmpty() && end - left.peek().getPosition() > 1) {
if (end > 0 && end < exp.length() - 1) {
char prev = exp.charAt(end - 1);
char next = exp.charAt(end + 1);
if (isCharWaitSepLeft(prev) && isCharWaitSepRight(next)) {
c = '*';// 替换成*,便于后续处理
}
}
}
}
// 处理<>内部空格===end
// find next operator
if (!OperatorUtil.isSeperator(c)) {
end++;
continue;
} else if (end > 0 && exp.charAt(end - 1) == '\\' && OperatorUtil.isOperator(c)) {// 转义符
exp = exp.replace(exp.charAt(end - 1), ' ');// 转义符用空格代替
end++;
continue;
}
// to-do:title 运算符转义
if (end > begin) {
String st = exp.substring(begin, end).trim();
if (!Strings.isNullOrEmpty(st)) {
v.add(st);
}
}
if (c != ',' && c != '，' && c != '"' && c != '“' && c != '”') {
v.add(new Character(c));
} else {
logger.warn("miss ',' at:{}", end);
}
end++;
begin = end;
}
if (end == exp.length() && begin <= end - 1) {
if (begin == end - 1 && OperatorUtil.isSeperator(exp.charAt(begin))) {// last
// op
v.add(new Character(exp.charAt(begin)));
} else {
String sub = exp.substring(begin).trim();
if (!Strings.isNullOrEmpty(sub)) {
v.add(sub);
}
}
}
// 遍历分隔结束,开始处理特殊字符
for (int i = 0; i < v.size(); i++) {
Object o = v.get(i);
if ((o instanceof String && ((String) o).trim().equals(""))
|| o instanceof Character && ((char) o == '\u0000' || (char) o == '\n' || (char) o == '\t')) {
// v.remove(i);
} else if (i > 0 && i < v.size() - 1) {
if (o instanceof String && ((String) o).trim().contains(" ")) {
Object prev = v.get(i - 1);
Object next = v.get(i + 1);
if (prev instanceof Character && (char) prev == '<' && next instanceof Character
&& (char) next == '>') {
String[] arr = ((String) o).split(" ");
if (null != arr && arr.length == 2) {
v.set(i, arr[0]);
v.add(i, '*');// 《》中间空格用*代替，便于后续处理
v.add(i, arr[1]);
}
}
}
}
}
// 处理~10类型
for (int j = 0; j < v.size(); j++) {
Object obj = v.get(j);
if (obj instanceof String) {
if (j > 0) {
Object prv = v.get(j - 1);
// 处理~10类型后面的数字
if (prv instanceof Character && (char) prv == '~') {
Integer va = Integer.parseInt((String) obj);
if (null != va) {
v.set(j, va);
}
continue;
}
}
// title:表达式
ExpObject ebj = null;
if (((String) obj).equals("title:") || ((String) obj).equals("content:")) {
v.remove(j);
if (j < v.size() - 1) {
Object next = v.get(j);// next now is j
if (next instanceof Character && (char) next == '(' || (char) next == '（') {
Stack<HalfQuartChar> stk = new Stack<HalfQuartChar>();
stk.push(new HalfQuartChar('(', j));
for (int k = j + 1; k < v.size(); k++) {
Object kv = v.get(k);
if (kv instanceof Character) {
if ((char) kv == '(' || (char) kv == '（') {
stk.push(new HalfQuartChar('(', k));
} else if ((char) kv == ')' || (char) kv == '）') {
if (!stk.isEmpty()) {
stk.pop();
}
if (stk.isEmpty()) {
break;
}
}
} else if (kv instanceof String) {
if (((String) obj).equals("title:")) {
ebj = new ExpObject((String) kv, new String[]{"title"});
} else {
ebj = new ExpObject((String) kv, fields);
}
v.set(k, ebj);
}
}
}
}
} else if (((String) obj).startsWith("title:") || ((String) obj).startsWith("content:")) {
if (((String) obj).startsWith("title:")) {
String value = ((String) obj).substring("title:".length());
ebj = new ExpObject((String) value, new String[]{"title"});
} else {
String value = ((String) obj).substring("content:".length());
ebj = new ExpObject((String) value, fields);
}
v.set(j, ebj);
} else {
List<String> fieldsNew = new ArrayList<>();
fieldsNew.add(ESConstant.TITLE);
List<String> fieldsList = Arrays.asList(fields);
fieldsNew.addAll(fieldsList);
ebj = new ExpObject((String) obj, fieldsNew.toArray(new String[fieldsNew.size()]));
v.set(j, ebj);
}
}
}
return v;
}
/**
* 表达式预处理 不自动添加title属性
*/
public static Vector<Object> preTreatExpressionNoTitleField(String exp, String[] fields) {
if (Strings.isNullOrEmpty(exp)) {
return null;
}
exp = exp.trim();
Vector<Object> v = new Vector<Object>();
Stack<HalfQuartChar> left = new Stack<>();// '<'
Stack<HalfQuartChar> leftQ = new Stack<>();// '('
Stack<HalfQuartChar> leftS = new Stack<>();// “ 或"
int begin = 0, end = 0;
while (end < exp.length()) {
char c = exp.charAt(end);
// 处理<>内部空格====begin
if (c == '<' || c == '《') {
left.push(new HalfQuartChar(c, end));
} else if (c == '》' || c == '>') {
if (!left.isEmpty()) {
left.pop();
}
} else if (c == '(' || c == '（') {
leftQ.push(new HalfQuartChar(c, end));
} else if (c == ')' || c == '）') {
if (!leftQ.isEmpty()) {
leftQ.pop();
}
} else if (c == '"' || c == '“') {
leftS.push(new HalfQuartChar(c, end));
} else if (c == '"' || c == '”') {
if (!leftS.isEmpty()) {
leftS.pop();
}
}
// 可能是<(A) (B)>,(A)和(B)之间的空格
if (c == ' ' && leftS.isEmpty()) {// 没有在“”内部
if (!left.isEmpty() && end - left.peek().getPosition() > 1) {
if (end > 0 && end < exp.length() - 1) {
char prev = exp.charAt(end - 1);
char next = exp.charAt(end + 1);
if (isCharWaitSepLeft(prev) && isCharWaitSepRight(next)) {
c = '*';// 替换成*,便于后续处理
}
}
}
}
// 处理<>内部空格===end
// find next operator
if (!OperatorUtil.isSeperator(c)) {
end++;
continue;
} else if (end > 0 && exp.charAt(end - 1) == '\\' && OperatorUtil.isOperator(c)) {// 转义符
exp = exp.replace(exp.charAt(end - 1), ' ');// 转义符用空格代替
end++;
continue;
}
// to-do:title 运算符转义
if (end > begin) {
String st = exp.substring(begin, end).trim();
if (!Strings.isNullOrEmpty(st)) {
v.add(st);
}
}
if (c != ',' && c != '，' && c != '"' && c != '“' && c != '”') {
v.add(new Character(c));
} else {
logger.warn("miss ',' at:{}", end);
}
end++;
begin = end;
}
if (end == exp.length() && begin <= end - 1) {
if (begin == end - 1 && OperatorUtil.isSeperator(exp.charAt(begin))) {// last
// op
v.add(new Character(exp.charAt(begin)));
} else {
String sub = exp.substring(begin).trim();
if (!Strings.isNullOrEmpty(sub)) {
v.add(sub);
}
}
}
// 遍历分隔结束,开始处理特殊字符
for (int i = 0; i < v.size(); i++) {
Object o = v.get(i);
if ((o instanceof String && ((String) o).trim().equals(""))
|| o instanceof Character && ((char) o == '\u0000' || (char) o == '\n' || (char) o == '\t')) {
// v.remove(i);
} else if (i > 0 && i < v.size() - 1) {
if (o instanceof String && ((String) o).trim().contains(" ")) {
Object prev = v.get(i - 1);
Object next = v.get(i + 1);
if (prev instanceof Character && (char) prev == '<' && next instanceof Character
&& (char) next == '>') {
String[] arr = ((String) o).split(" ");
if (null != arr && arr.length == 2) {
v.set(i, arr[0]);
v.add(i, '*');// 《》中间空格用*代替，便于后续处理
v.add(i, arr[1]);
}
}
}
}
}
// 处理~10类型
for (int j = 0; j < v.size(); j++) {
Object obj = v.get(j);
if (obj instanceof String) {
if (j > 0) {
Object prv = v.get(j - 1);
// 处理~10类型后面的数字
if (prv instanceof Character && (char) prv == '~') {
Integer va = Integer.parseInt((String) obj);
if (null != va) {
v.set(j, va);
}
continue;
}
}
// title:表达式
ExpObject ebj = null;
if (((String) obj).equals("title:") || ((String) obj).equals("content:")) {
v.remove(j);
if (j < v.size() - 1) {
Object next = v.get(j);// next now is j
if (next instanceof Character && (char) next == '(' || (char) next == '（') {
Stack<HalfQuartChar> stk = new Stack<HalfQuartChar>();
stk.push(new HalfQuartChar('(', j));
for (int k = j + 1; k < v.size(); k++) {
Object kv = v.get(k);
if (kv instanceof Character) {
if ((char) kv == '(' || (char) kv == '（') {
stk.push(new HalfQuartChar('(', k));
} else if ((char) kv == ')' || (char) kv == '）') {
if (!stk.isEmpty()) {
stk.pop();
}
if (stk.isEmpty()) {
break;
}
}
} else if (kv instanceof String) {
if (((String) obj).equals("title:")) {
ebj = new ExpObject((String) kv, new String[]{"title"});
} else {
ebj = new ExpObject((String) kv, fields);
}
v.set(k, ebj);
}
}
}
}
} else if (((String) obj).startsWith("title:") || ((String) obj).startsWith("content:")) {
if (((String) obj).startsWith("title:")) {
String value = ((String) obj).substring("title:".length());
ebj = new ExpObject((String) value, new String[]{"title"});
} else {
String value = ((String) obj).substring("content:".length());
ebj = new ExpObject((String) value, fields);
}
v.set(j, ebj);
} else {
List<String> fieldsNew = new ArrayList<>();
//fieldsNew.add(ESConstant.TITLE);
List<String> fieldsList = Arrays.asList(fields);
fieldsNew.addAll(fieldsList);
ebj = new ExpObject((String) obj, fieldsNew.toArray(new String[fieldsNew.size()]));
v.set(j, ebj);
}
}
}
return v;
}
public static Vector<Object> preProcessExpression(String exp, boolean onlyKeyWord, String[] fields) throws Exception {
if (Strings.isNullOrEmpty(exp)) {
return null;
}
exp = exp.trim();
String[] arr = exp.split(" ");
Vector<Object> v = new Vector<>();
int quartNum = 0;
for (String st : arr) {
st = st.trim();
if (Strings.isNullOrEmpty(st)) {
continue;
}
int begin = 0, end = 0;
while (end < st.length()) {
char c = st.charAt(end);
// find next operator
if (!OperatorUtil.isSeperator(c)) {
end++;
continue;
} else { // c为分隔符
if (end > 0 && st.charAt(end - 1) == '\\') {// 转义符
st = st.replace(st.charAt(end - 1), ' ');// 转义符用空格代替
end++;
continue;
} else if (c == '“' || (c == '"' && quartNum % 2 == 0)) {
quartNum++;
} else if (c == '”' || (c == '"' && quartNum % 2 == 1)) {
quartNum--;
} else if (quartNum > 0) {
end++;
continue;
}
}
if (end > begin) {
String st0 = st.substring(begin, end).trim();
if (!Strings.isNullOrEmpty(st0)) {
v.add(st0);
}
}
if (c != ',' && c != '，') {
v.add(new Character(c));
} else {
logger.warn("miss ',' at:{}", end);
}
end++;
begin = end;
}
if (end == st.length() && begin <= end - 1) {
if (begin == end - 1 && OperatorUtil.isSeperator(st.charAt(begin)) && quartNum == 0) {
v.add(new Character(st.charAt(begin)));
} else {
String sub = st.substring(begin).trim();
if (!Strings.isNullOrEmpty(sub)) {
v.add(sub);
}
}
}
}
logger.debug("after split,vector now is:{}",v);
visitVector(v);
logger.info("=====================endLess loop=============");
logger.info("keyWord is:{}", exp);
v = checkAndFormatVector(v, onlyKeyWord, fields);
logger.debug("after checkAndFormatVector,vector now is:{}",v);
visitVector(v);
return v;
}
public static Vector<Object> preProcessExpressionNoTitleField(String exp, boolean onlyKeyWord, String[] fields) throws Exception {
if (Strings.isNullOrEmpty(exp)) {
return null;
}
exp = exp.trim();
String[] arr = exp.split(" ");
Vector<Object> v = new Vector<>();
int quartNum = 0;
for (String st : arr) {
st = st.trim();
if (Strings.isNullOrEmpty(st)) {
continue;
}
int begin = 0, end = 0;
while (end < st.length()) {
char c = st.charAt(end);
// find next operator
if (!OperatorUtil.isSeperator(c)) {
end++;
continue;
} else { // c为分隔符
if (end > 0 && st.charAt(end - 1) == '\\') {// 转义符
st = st.replace(st.charAt(end - 1), ' ');// 转义符用空格代替
end++;
continue;
} else if (c == '“' || (c == '"' && quartNum % 2 == 0)) {
quartNum++;
} else if (c == '”' || (c == '"' && quartNum % 2 == 1)) {
quartNum--;
} else if (quartNum > 0) {
end++;
continue;
}
}
if (end > begin) {
String st0 = st.substring(begin, end).trim();
if (!Strings.isNullOrEmpty(st0)) {
v.add(st0);
}
}
if (c != ',' && c != '，') {
v.add(new Character(c));
} else {
logger.warn("miss ',' at:{}", end);
}
end++;
begin = end;
}
if (end == st.length() && begin <= end - 1) {
if (begin == end - 1 && OperatorUtil.isSeperator(st.charAt(begin)) && quartNum == 0) {
v.add(new Character(st.charAt(begin)));
} else {
String sub = st.substring(begin).trim();
if (!Strings.isNullOrEmpty(sub)) {
v.add(sub);
}
}
}
}
logger.debug("after split,vector now is:{}",v);
visitVector(v);
logger.info("=====================endLess loop=============");
logger.info("keyWord is:{}", exp);
v = checkAndFormatVectorNoTitleField(v, onlyKeyWord, fields);
logger.debug("after checkAndFormatVector,vector now is:{}",v);
visitVector(v);
return v;
}
private static void visitVector(Vector<Object> vector) {
if (null == vector || vector.isEmpty()) {
logger.debug("vector is empty!");
return;
}
for (int i = 0; i < vector.size(); i++) {
Object obj = vector.get(i);
logger.debug("i=" + i + ",value=" + obj);
}
return;
}
private static Vector<Object> checkAndFormatVector(Vector<Object> vector, boolean onlyKeyWord, String[] fields) throws Exception {
if (null == vector || vector.isEmpty()) {
return null;
}
logger.warn("[checkAndFormatVector] fields: {}{}", fields);
List list = new ArrayList();
for (int i = 0; i < vector.size(); i++) {
list.add(vector.get(i));
}
logger.info("vector list is:{}", list);
Stack<HalfQuartChar> leftS = new Stack<HalfQuartChar>();// 存放“”
Stack<HalfQuartChar> leftQ = new Stack<HalfQuartChar>();// 存放<>
int qNum = 0;// <>数目
int starNum = 0;// <>内部分隔空格个数
for (int i = 0; i < vector.size(); i++) {
Object ob = vector.get(i);
// <>内部
if (!leftQ.isEmpty() && leftS.isEmpty()) {
if (i > 0) {
if (isLeftWaitSeperate(vector.get(i - 1)) && isRightWaitSeperate(vector.get(i))) {
vector.insertElementAt(new Character('*'), i);
starNum++;
continue;
} else if (i > 1 && isLeftWaitSeperate(vector.get(i - 2))
&& isRightWaitSeperate(vector.get(i - 1))) {
vector.insertElementAt(new Character('*'), i - 1);
starNum++;
continue;
}
}
}
if (ob instanceof Character) {
if ((char) ob == '“' || ((char) ob == '"' && leftS.size() % 2 == 0)) {
leftS.push(new HalfQuartChar('“', i));
vector.remove(ob);
i--;
} else if ((char) ob == '”' || ((char) ob == '"' && leftS.size() % 2 == 1)) {
if (!leftS.isEmpty()) {
leftS.pop();
} else {
throw new Exception(" “” not match Exception,“ expected ");
}
vector.remove(ob);
i--;
} else if ((char) ob == '<' || (char) ob == '《') {
leftQ.push(new HalfQuartChar('<', i));
qNum++;
} else if ((char) ob == '>' || (char) ob == '》') {
if (!leftQ.isEmpty()) {
leftQ.pop();
} else {
throw new Exception("> not match Exception,< expected ");
}
}
} else if (ob instanceof String) {
if (i > 0) {
Object prv = vector.get(i - 1);
if (prv instanceof String && onlyKeyWord) {// 连续String
String ns = null;
if (!leftS.isEmpty()) {// “”内部连续String=>合并
ns = ((String) prv).trim() + " " + ((String) ob).trim();
} else if (leftQ.isEmpty()) {// 不在<>内部
ns = ((String) prv).trim() + ((String) ob).trim();
}
if (!Strings.isNullOrEmpty(ns)) {
vector.set(i - 1, ns);
vector.remove(i--);
}
}
}
}
}
// “”不匹配
if (!leftS.isEmpty()) {
throw new Exception(" “” not match Exception!");
} else if (!leftQ.isEmpty()) {
throw new Exception("<> not match Exception!");
} else if (starNum != qNum) {
throw new Exception("<> phrase Exception,riht format should like: <A B>~10");
}
// 封装成OP或者ExpObject
for (int j = 0; j < vector.size(); j++) {
Object obj = vector.get(j);
if (obj instanceof String) {
if (j > 0) {
Object prv = vector.get(j - 1);
// 处理~10类型后面的数字
if (prv instanceof Character && (char) prv == '~') {
Integer va = Integer.parseInt((String) obj);
if (null != va) {
vector.set(j, va);
}
continue;
}
}
// title:表达式
ExpObject ebj = null;
if (((String) obj).equals("title:") || ((String) obj).equals("content:")) {
vector.remove(j);
if (j < vector.size() - 1) {
Object next = vector.get(j);// next now is j
if (next instanceof Character && (char) next == '(' || (char) next == '（') {
Stack<HalfQuartChar> stk = new Stack<HalfQuartChar>();
stk.push(new HalfQuartChar('(', j));
for (int k = j + 1; k < vector.size(); k++) {
Object kv = vector.get(k);
if (kv instanceof Character) {
if ((char) kv == '(' || (char) kv == '（') {
stk.push(new HalfQuartChar('(', k));
} else if ((char) kv == ')' || (char) kv == '）') {
if (!stk.isEmpty()) {
stk.pop();
}
if (stk.isEmpty()) {
break;
}
}
} else if (kv instanceof String) {// 括号内部普通字符串
// 类似 title:(A|title:B)==》去掉title:B之前的title:
while (((String) kv).startsWith("title:") || ((String) kv).startsWith("content:")) {
int begin = ((String) kv).indexOf(':');
kv = ((String) kv).substring(begin + 1).trim();
}
if (((String) kv).startsWith("\\title:")// 使用转义符==>当做普通字符串处理
|| ((String) kv).startsWith("\\content:")) {
kv = ((String) kv).substring(2).trim();
}
if (((String) obj).equals("title:")) {
ebj = new ExpObject((String) kv, new String[]{"title"});
} else {
ebj = new ExpObject((String) kv, fields);
}
vector.set(k, ebj);
}
}
}
}
} else if (((String) obj).startsWith("title:") || ((String) obj).startsWith("content:")) {
String value = (String) obj;
while (value.startsWith("title:") || value.startsWith("content:")) {
value = value.substring(value.indexOf(":") + 1);
}
if (((String) obj).startsWith("title:")) {
ebj = new ExpObject((String) value, new String[]{"title"});
} else {
ebj = new ExpObject((String) value, fields);
}
vector.set(j, ebj);
} else {// 普通字符串
if (((String) obj).startsWith("\\title:")// 使用转义符==>当做普通字符串处理
|| ((String) obj).startsWith("\\content:")) {
obj = ((String) obj).substring(2).trim();
}
List<String> fieldsNew = new ArrayList<>();
fieldsNew.add(ESConstant.TITLE);
List<String> fieldsList = Arrays.asList(fields);
fieldsNew.addAll(fieldsList);
ebj = new ExpObject((String) obj, fieldsNew.toArray(new String[fieldsNew.size()]));
vector.set(j, ebj);
}
}
}
return vector;
}
private static Vector<Object> checkAndFormatVectorNoTitleField(Vector<Object> vector, boolean onlyKeyWord, String[] fields) throws Exception {
if (null == vector || vector.isEmpty()) {
return null;
}
logger.warn("[checkAndFormatVector] fields: {}{}", fields);
List list = new ArrayList();
for (int i = 0; i < vector.size(); i++) {
list.add(vector.get(i));
}
logger.info("vector list is:{}", list);
Stack<HalfQuartChar> leftS = new Stack<HalfQuartChar>();// 存放“”
Stack<HalfQuartChar> leftQ = new Stack<HalfQuartChar>();// 存放<>
int qNum = 0;// <>数目
int starNum = 0;// <>内部分隔空格个数
for (int i = 0; i < vector.size(); i++) {
Object ob = vector.get(i);
// <>内部
if (!leftQ.isEmpty() && leftS.isEmpty()) {
if (i > 0) {
if (isLeftWaitSeperate(vector.get(i - 1)) && isRightWaitSeperate(vector.get(i))) {
vector.insertElementAt(new Character('*'), i);
starNum++;
continue;
} else if (i > 1 && isLeftWaitSeperate(vector.get(i - 2))
&& isRightWaitSeperate(vector.get(i - 1))) {
vector.insertElementAt(new Character('*'), i - 1);
starNum++;
continue;
}
}
}
if (ob instanceof Character) {
if ((char) ob == '“' || ((char) ob == '"' && leftS.size() % 2 == 0)) {
leftS.push(new HalfQuartChar('“', i));
vector.remove(ob);
i--;
} else if ((char) ob == '”' || ((char) ob == '"' && leftS.size() % 2 == 1)) {
if (!leftS.isEmpty()) {
leftS.pop();
} else {
throw new Exception(" “” not match Exception,“ expected ");
}
vector.remove(ob);
i--;
} else if ((char) ob == '<' || (char) ob == '《') {
leftQ.push(new HalfQuartChar('<', i));
qNum++;
} else if ((char) ob == '>' || (char) ob == '》') {
if (!leftQ.isEmpty()) {
leftQ.pop();
} else {
throw new Exception("> not match Exception,< expected ");
}
}
} else if (ob instanceof String) {
if (i > 0) {
Object prv = vector.get(i - 1);
if (prv instanceof String && onlyKeyWord) {// 连续String
String ns = null;
if (!leftS.isEmpty()) {// “”内部连续String=>合并
ns = ((String) prv).trim() + " " + ((String) ob).trim();
} else if (leftQ.isEmpty()) {// 不在<>内部
ns = ((String) prv).trim() + ((String) ob).trim();
}
if (!Strings.isNullOrEmpty(ns)) {
vector.set(i - 1, ns);
vector.remove(i--);
}
}
}
}
}
// “”不匹配
if (!leftS.isEmpty()) {
throw new Exception(" “” not match Exception!");
} else if (!leftQ.isEmpty()) {
throw new Exception("<> not match Exception!");
} else if (starNum != qNum) {
throw new Exception("<> phrase Exception,riht format should like: <A B>~10");
}
// 封装成OP或者ExpObject
for (int j = 0; j < vector.size(); j++) {
Object obj = vector.get(j);
if (obj instanceof String) {
if (j > 0) {
Object prv = vector.get(j - 1);
// 处理~10类型后面的数字
if (prv instanceof Character && (char) prv == '~') {
Integer va = Integer.parseInt((String) obj);
if (null != va) {
vector.set(j, va);
}
continue;
}
}
// title:表达式
ExpObject ebj = null;
if (((String) obj).equals("title:") || ((String) obj).equals("content:")) {
vector.remove(j);
if (j < vector.size() - 1) {
Object next = vector.get(j);// next now is j
if (next instanceof Character && (char) next == '(' || (char) next == '（') {
Stack<HalfQuartChar> stk = new Stack<HalfQuartChar>();
stk.push(new HalfQuartChar('(', j));
for (int k = j + 1; k < vector.size(); k++) {
Object kv = vector.get(k);
if (kv instanceof Character) {
if ((char) kv == '(' || (char) kv == '（') {
stk.push(new HalfQuartChar('(', k));
} else if ((char) kv == ')' || (char) kv == '）') {
if (!stk.isEmpty()) {
stk.pop();
}
if (stk.isEmpty()) {
break;
}
}
} else if (kv instanceof String) {// 括号内部普通字符串
// 类似 title:(A|title:B)==》去掉title:B之前的title:
while (((String) kv).startsWith("title:") || ((String) kv).startsWith("content:")) {
int begin = ((String) kv).indexOf(':');
kv = ((String) kv).substring(begin + 1).trim();
}
if (((String) kv).startsWith("\\title:")// 使用转义符==>当做普通字符串处理
|| ((String) kv).startsWith("\\content:")) {
kv = ((String) kv).substring(2).trim();
}
if (((String) obj).equals("title:")) {
ebj = new ExpObject((String) kv, new String[]{"title"});
} else {
ebj = new ExpObject((String) kv, fields);
}
vector.set(k, ebj);
}
}
}
}
} else if (((String) obj).startsWith("title:") || ((String) obj).startsWith("content:")) {
String value = (String) obj;
while (value.startsWith("title:") || value.startsWith("content:")) {
value = value.substring(value.indexOf(":") + 1);
}
if (((String) obj).startsWith("title:")) {
ebj = new ExpObject((String) value, new String[]{"title"});
} else {
ebj = new ExpObject((String) value, fields);
}
vector.set(j, ebj);
} else {// 普通字符串
if (((String) obj).startsWith("\\title:")// 使用转义符==>当做普通字符串处理
|| ((String) obj).startsWith("\\content:")) {
obj = ((String) obj).substring(2).trim();
}
List<String> fieldsNew = new ArrayList<>();
//fieldsNew.add(ESConstant.TITLE);
List<String> fieldsList = Arrays.asList(fields);
fieldsNew.addAll(fieldsList);
ebj = new ExpObject((String) obj, fieldsNew.toArray(new String[fieldsNew.size()]));
vector.set(j, ebj);
}
}
}
return vector;
}
/**
* 处理消歧语句表达式
*/
public static HashSet<ExpObject> handleMinusExpression(String minus, String[] fields) throws Exception {
if (Strings.isNullOrEmpty(minus)) {
return null;
}
minus = minus.replaceAll("，", ",");
minus = minus.replaceAll(" ", " ");
Vector<Object> v = preProcessExpression(minus, false, fields);
HashSet<ExpObject> set = new HashSet<ExpObject>();
if (null != v && !v.isEmpty()) {
for (Object obj : v) {
if (obj instanceof ExpObject) {
set.add((ExpObject) obj);
}
}
}
return set;
}
private static boolean isCharWaitSepLeft(char charC) {
if (!OperatorUtil.isSeperator(charC) || charC == ')' || charC == '）' || charC == '>' || charC == '》'
|| charC == '"' || charC == '”') {
return true;
}
return false;
}
private static boolean isCharWaitSepRight(char charC) {
if (!OperatorUtil.isSeperator(charC) || charC == '(' || charC == '（' || charC == '<' || charC == '《'
|| charC == '"' || charC == '“') {
return true;
}
return false;
}
private static boolean isLeftWaitSeperate(Object obj) {
if (null == obj) {
return false;
}
if (obj instanceof Character) {
return (char) obj == ')' || (char) obj == '）' || (char) obj == '"' || (char) obj == '”';
} else if (obj instanceof String) {
return !Strings.isNullOrEmpty((String) obj);
} else if (obj instanceof Integer) {// ~20
return true;
}
return false;
}
private static boolean isRightWaitSeperate(Object obj) {
if (null == obj) {
return false;
}
if (obj instanceof Character) {
return (char) obj == '(' || (char) obj == '（' || (char) obj == '<' || (char) obj == '《' || (char) obj == '"'
|| (char) obj == '“';
} else if (obj instanceof String) {
return !Strings.isNullOrEmpty((String) obj);
}
return false;
}
}

669
cl_search_api/src/main/java/com/bfd/mf/common/util/subject/QueryBuilderUtil.java
View File

@ -1,669 +0,0 @@
package com.bfd.mf.common.util.subject;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.bfd.mf.common.util.constants.ESConstant;
import com.bfd.mf.common.util.ESServerUtils;
import com.bfd.mf.common.web.repository.mysql.cache.ClusterRepository;
import com.bfd.mf.common.web.vo.params.expression.ExpNode;
import com.bfd.mf.common.web.vo.params.expression.ExpObject;
import com.bfd.nlp.common.util.object.TObjectUtils;
import com.bfd.nlp.common.util.string.TStringUtils;
import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse;
import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.AnalyzeToken;
import org.elasticsearch.client.IndicesAdminClient;
import org.elasticsearch.common.Strings;
import org.elasticsearch.index.query.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import java.util.*;
import static com.bfd.mf.common.util.constants.ESConstant.EXCLUDE_KEYWORDS_SPLIT_CN_SYSBOL;
import static com.bfd.mf.common.util.constants.ESConstant.EXCLUDE_KEYWORDS_SPLIT_SYSBOL;
public class QueryBuilderUtil {
private static final Logger logger = LoggerFactory.getLogger(QueryBuilderUtil.class);
@Autowired
private ClusterRepository clusterRepository;
public static BoolQueryBuilder getQueryBuilderFromExpNodeTree(ExpNode root, HashSet<ExpObject> set) throws Exception {
if (null == root || null == root.getValue()) {
return null;
}
/*
* @2016-12-7
* @author lei.bao
* format之后:
* 1:distance运算拆成全部单个(左右子树全部是ExpObject)
* 2:所有ExpObject都是叶子节点且规范
*/
ExpNode rst = ExpNodeUtil.visitAndFormatTree(root, set);
//
Object rt = ExpNodeUtil.ComputeNodeTree(rst, set);
if (rt instanceof BoolQueryBuilder) {
return (BoolQueryBuilder) rt;
} else if (rt instanceof MatchQueryBuilder) {
BoolQueryBuilder bq = QueryBuilders.boolQuery();
return bq.must((MatchQueryBuilder) rt);
}else if(rt instanceof MatchPhraseQueryBuilder){
BoolQueryBuilder bq = QueryBuilders.boolQuery();
return bq.must((MatchPhraseQueryBuilder) rt);
}else if(rt instanceof AbstractQueryBuilder){
BoolQueryBuilder bq = QueryBuilders.boolQuery();
return bq.must((AbstractQueryBuilder) rt);
}
return null;
}
public static BoolQueryBuilder handleExcludeExpression(String exclude, String[] fields) {
Vector<Object> v = ExpressionParser.preTreatExpression(exclude, fields);
if (null != v && !v.isEmpty()) {
BoolQueryBuilder qb = QueryBuilders.boolQuery();
for (Object obj : v) {
if (obj instanceof ExpObject) {
BoolQueryBuilder innerQb = QueryBuilders.boolQuery();
for (String fd : ((ExpObject) obj).getFields()) {
innerQb.should(QueryBuilders.matchPhraseQuery(fd, ((ExpObject) obj).getExpression()).slop(0));
}
qb.mustNot(innerQb);
}
}
return qb;
}
return null;
}
public static BoolQueryBuilder handleExcludeExpressionNoTitleField(String exclude, String[] fields) {
Vector<Object> v = ExpressionParser.preTreatExpressionNoTitleField(exclude, fields);
if (null != v && !v.isEmpty()) {
BoolQueryBuilder qb = QueryBuilders.boolQuery();
for (Object obj : v) {
if (obj instanceof ExpObject) {
BoolQueryBuilder innerQb = QueryBuilders.boolQuery();
for (String fd : ((ExpObject) obj).getFields()) {
innerQb.should(QueryBuilders.matchPhraseQuery(fd, ((ExpObject) obj).getExpression()).slop(0));
}
qb.mustNot(innerQb);
}
}
return qb;
}
return null;
}
public static BoolQueryBuilder handleTitleExcludeExpression(String exclude) {
if (TStringUtils.isNotEmpty(exclude)) {
// 标点符号修正
exclude = exclude.replaceAll(EXCLUDE_KEYWORDS_SPLIT_CN_SYSBOL,
EXCLUDE_KEYWORDS_SPLIT_SYSBOL);
exclude = "title:(" + exclude + ")";
Vector<Object> v = ExpressionParser.preTreatExpression(exclude, new String[]{ESConstant.TITLE});
BoolQueryBuilder qb = QueryBuilders.boolQuery();
BoolQueryBuilder titleExcludeBuilder = null;
if (null != v && !v.isEmpty()) {
titleExcludeBuilder = QueryBuilders.boolQuery();
for (Object obj : v) {
if (obj instanceof ExpObject) {
QueryBuilder wordQueryBuilder = QueryBuilders.matchPhraseQuery(ESConstant.SEARCH_SCOPE_TYPE_TITLE,
((ExpObject) obj).getExpression()).slop(0);
titleExcludeBuilder.should(wordQueryBuilder);
}
}
qb.mustNot(titleExcludeBuilder);
return qb;
}
}
return null;
}
public static List<SpanNotQueryBuilder> getSpanNotQueryBuilder(String field, String shortStr, String longerStr) {
if (Strings.isNullOrEmpty(shortStr) || Strings.isNullOrEmpty(longerStr)) {
logger.error("empty short term or longer term,return NULL");
return null;
}
int start = longerStr.indexOf(shortStr);
if (start < 0 || start >= longerStr.length()) {
return null;
}
int lenS = shortStr.length();
if ((lenS + start) > longerStr.length()) {
return null;
}
List<String> subs = new ArrayList<String>();
if (start > 0) {
subs.add(longerStr.substring(0, start));
}
subs.add(longerStr.substring(start, start + lenS));
if ((start + lenS) < longerStr.length()) {
subs.add(longerStr.substring(start + lenS));
}
// call API to get include clause
AnalyzeResult anaLyRst = getAnalyzeResult(shortStr);
if (null == anaLyRst) {
return null;
}
List<String> rst = anaLyRst.getList();
// include clause
List<SpanNearQueryBuilder> spans = new ArrayList<>();
SpanNearQueryBuilder span;
if (null != rst && rst.size() > 1) {
for (String tk : rst) {
span = new SpanNearQueryBuilder(
new SpanTermQueryBuilder(field, tk), anaLyRst.getSlop()).inOrder(true);
spans.add(span);
}
} else {
span = new SpanNearQueryBuilder(
new SpanTermQueryBuilder(field, shortStr), anaLyRst.getSlop()).inOrder(true);
spans.add(span);
}
List<String> lst = new LinkedList<>();
int maxDis = 0;
for (String st : subs) {
AnalyzeResult anLyRst = getAnalyzeResult(st);
if (null == anLyRst) {
continue;
}
if (anLyRst.getSlop() > maxDis) {
maxDis = anLyRst.getSlop();
}
List<String> rt = anLyRst.getList();
for (String s : rt) {
if (!listContanins(lst, s)) {
lst.add(s);
}
}
}
List<SpanNotQueryBuilder> sns = new ArrayList<>();
SpanNotQueryBuilder sn = null;
if (!lst.isEmpty()) {
List<SpanNearQueryBuilder> sners = new ArrayList<>();
SpanNearQueryBuilder sner = null;
for (String st0 : lst) {
sner = new SpanNearQueryBuilder(new SpanTermQueryBuilder(field, st0), maxDis).inOrder(true);
sners.add(sner);
}
if (spans.size() < sners.size()) {
for (SpanNearQueryBuilder spanNearQueryBuilder : sners) {
for (SpanNearQueryBuilder span1 : spans) {
sn = new SpanNotQueryBuilder(span1, spanNearQueryBuilder);
sns.add(sn);
}
}
}
}
return sns;
}
public static String getBoolQueryBuilderStr(BoolQueryBuilder qb) {
String st = qb.toString();
char[] cs = st.toCharArray();
StringBuilder sb = new StringBuilder();
for (char c : cs) {
if (c != '\u0000' && c != '\n' && c != '\t') {
sb.append(c);
}
}
String sb1 = sb.toString().replaceAll(" ", " ");
while (sb1.contains(" ")) {
sb1 = sb1.replaceAll(" ", " ");
}
String[] st0 = sb1.toString().split(" ");
sb = new StringBuilder();
int num = 0;
for (String s0 : st0) {
if (!Strings.isNullOrEmpty(s0)) {
while (s0.startsWith(" ") || s0.endsWith(" ")) {
s0 = s0.trim();
}
if (s0.startsWith("\"") || s0.startsWith("\\“")) {
num++;
} else if (s0.endsWith("\"") || s0.endsWith("\\”")) {
num--;
}
if (!"".equals(s0)) {
sb.append(s0);
}
if (num > 0) {
sb.append(" ");
}
}
}
st = "BoolQueryBuilder=" + sb.toString();
return st;
}
public static BoolQueryBuilder getBoolQueryBuilderFromSqlStr(String str) {
BoolQueryBuilder qb = QueryBuilders.boolQuery();
if (Strings.isNullOrEmpty(str) || !str.startsWith("BoolQueryBuilder=")) {
return qb;
}
// str = str.replaceAll("\t", " ");
int start = str.indexOf("{");
String jsonStr = str.substring(start);
JSONObject json = (JSONObject) JSONObject.parse(jsonStr);
if (null == json) {
return qb;
}
// QueryBuilder pqT = getQueryBuilderFromJSON(json);
// if (null != pqT && pqT instanceof BoolQueryBuilder) {
// return (BoolQueryBuilder) pqT;
// }
WrapperQueryBuilder wrapper = QueryBuilders.wrapperQuery(jsonStr);
qb.must(wrapper);
return qb;
}
private static List<QueryBuilder> getQueryBuilderFromJSON(JSONObject json) {
if (null == json) {
return null;
}
JSONObject bool = (JSONObject) json.get("bool");
if (TObjectUtils.isNull(bool)) {
// add term and so on
JSONObject query = (JSONObject) json.get("query_string");
if (null != query) {
String opr = query.getString("default_operator");
Operator op = null;
if (null != opr && opr.equalsIgnoreCase("and")) {
op = Operator.AND;
} else {
op = Operator.OR;
}
List<QueryBuilder> tqbs = new ArrayList<>();
QueryBuilder tqb = new QueryStringQueryBuilder(query.getString("query"))
.defaultField(query.getString("default_field")).defaultOperator(op);
tqbs.add(tqb);
return tqbs;
}
JSONObject term = (JSONObject) json.get("term");
if (null != term) {
List<QueryBuilder> tqbs = new ArrayList<>();
QueryBuilder tqb = null;
if (null != term.get("content")) {
tqb = new TermQueryBuilder("content", term.get("content"));
} else if (null != term.get("title")) {
tqb = new TermQueryBuilder("title", term.get("title"));
} else if (null != term.get("source")) {
tqb = new TermQueryBuilder("source", term.get("source"));
}
tqbs.add(tqb);
return tqbs;
}
// match
JSONObject match = (JSONObject) json.get("match");
if (null != match) {
JSONObject content = match.getJSONObject("content");
if (null != content) {
List<QueryBuilder> list = new ArrayList<>();
QueryBuilder slop = QueryBuilders
.matchPhraseQuery("content", content.getString("query"))
.slop(content.getIntValue("slop"));
list.add(slop);
return list;
} else {
JSONObject title = match.getJSONObject("title");
if (null != title) {
List<QueryBuilder> list = new ArrayList<>();
QueryBuilder slop = QueryBuilders.matchPhraseQuery("title", title.getString("query"))
.slop(title.getIntValue("slop"));
list.add(slop);
return list;
}
}
}
// multi_match
JSONObject multi_match = json.getJSONObject("multi_match");
if (null != multi_match) {
String qry = multi_match.getString("query");
JSONArray fields = (JSONArray) multi_match.get("fields");
String[] arr = new String[fields.size()];//
for (int i = 0; i < fields.size(); i++) {
Object f = fields.get(i);
if (f instanceof String) {
arr[i] = (String) f;
}
}
int slop = multi_match.getIntValue("slop");
// MultiMatchQueryBuilder mt = new MultiMatchQueryBuilder();
List<QueryBuilder> list = new ArrayList<>();
QueryBuilder multiQuery = QueryBuilders.multiMatchQuery(qry, arr).slop(slop);
list.add(multiQuery);
return list;
}
// span_term
JSONObject span_term = (JSONObject) json.getJSONObject("span_term");
if (null != span_term) {
JSONObject content = span_term.getJSONObject("content");
if (null != content) {
List<QueryBuilder> list = new ArrayList<>();
QueryBuilder span_termQ = new SpanTermQueryBuilder("content", content.getString("value"));
// spanNotQ.include(span_termQ);
list.add(span_termQ);
return list;
}
JSONObject title = span_term.getJSONObject("title");
if (null != title) {
List<QueryBuilder> list = new ArrayList<>();
QueryBuilder span_termQ = new SpanTermQueryBuilder("title", title.getString("value"));
list.add(span_termQ);
return list;
}
}
// span_not
JSONObject span_not = (JSONObject) json.get("span_not");
List<QueryBuilder> span_termQ = null;
if (null != span_not) {
List<QueryBuilder> spanNotQs = new ArrayList<>();
SpanNotQueryBuilder spanNotQ;
JSONObject include = span_not.getJSONObject("include");
SpanNearQueryBuilder sner = null;
if (null != include) {
span_termQ = getQueryBuilderFromJSON(include);
JSONObject span_near = include.getJSONObject("span_near");
if (null != span_near && span_near.size() > 0) {
Object array = span_near.get("clauses");
if (null != array) {
if (array instanceof JSONObject) {
List<QueryBuilder> span_termQ2 = getQueryBuilderFromJSON((JSONObject) array);
if (null != span_termQ2 && span_termQ2.size() > 0 &&
span_termQ2 instanceof SpanTermQueryBuilder) {
sner = new SpanNearQueryBuilder((SpanQueryBuilder) span_termQ2.get(0), 0).inOrder(true);
}
} else if (array instanceof JSONArray) {
ListIterator<Object> arr = ((JSONArray) array).listIterator();
while (arr.hasNext()) {
Object obj = arr.next();
if (null == obj)
continue;
JSONObject jsn = (JSONObject) obj;
List<QueryBuilder> span_termQ2 = getQueryBuilderFromJSON((JSONObject) jsn);
if (null != span_termQ2 && span_termQ2.size() > 0 && span_termQ2 instanceof SpanTermQueryBuilder) {
sner = new SpanNearQueryBuilder((SpanQueryBuilder) span_termQ2.get(0), 0)
.inOrder(true);
}
}
}
}
}
}
JSONObject exclude = span_not.getJSONObject("exclude");
SpanNearQueryBuilder snerExclude = null;
if (null != exclude) {
JSONObject span_near = exclude.getJSONObject("span_near");
if (null != span_near) {
Object array = span_near.get("clauses");
if (null != array) {
if (array instanceof JSONObject) {
List<QueryBuilder> span_termQs = getQueryBuilderFromJSON((JSONObject) array);
if (null != span_termQs && span_termQs.size() > 0 && span_termQs instanceof SpanTermQueryBuilder) {
// spanNotQ..exclude((SpanTermQueryBuilder)
// span_termQ);
snerExclude = new SpanNearQueryBuilder((SpanQueryBuilder) span_termQs.get(0), 0).inOrder(true);
}
} else if (array instanceof JSONArray) {
ListIterator<Object> arr = ((JSONArray) array).listIterator();
while (arr.hasNext()) {
Object obj = arr.next();
if (null == obj)
continue;
JSONObject jsn = (JSONObject) obj;
List<QueryBuilder> span_termQs = getQueryBuilderFromJSON((JSONObject) jsn);
if (null != span_termQs && span_termQs.size() > 0 && span_termQs instanceof SpanTermQueryBuilder) {
// spanNotQ.exclude((SpanTermQueryBuilder)
// span_termQ);
snerExclude = new SpanNearQueryBuilder((SpanQueryBuilder) span_termQs.get(0), 0).inOrder(true);
}
}
}
}
}
}
if (span_termQ != null && snerExclude != null) {
spanNotQ = new SpanNotQueryBuilder((SpanQueryBuilder) span_termQ, snerExclude);
spanNotQs.add(spanNotQ);
}
if (sner != null && snerExclude != null) {
spanNotQ = new SpanNotQueryBuilder(sner, snerExclude);
spanNotQs.add(spanNotQ);
}
return spanNotQs;
}
return null;
}
// add child BoolQueryBuilder
List<QueryBuilder> pqbs = new ArrayList<>();
BoolQueryBuilder pqb = new BoolQueryBuilder();
//
Object filter = bool.get("filter");
if (filter instanceof JSONObject) {
JSONObject obj = (JSONObject) filter;
if (null != obj) {
List<QueryBuilder> qb = getQueryBuilderFromJSON(obj);
if (null != qb) {
pqb.filter(qb.get(0));
}
}
} else if (filter instanceof JSONArray) {
JSONArray array = (JSONArray) filter;
JSONObject[] objs = new JSONObject[array.size()];
array.toArray(objs);
for (JSONObject obj : objs) {
List<QueryBuilder> qb = getQueryBuilderFromJSON(obj);
if (null != qb) {
pqb.filter(qb.get(0));
}
}
}
pqbs.add(pqb);
//
Object mst = bool.get("must");
if (mst instanceof JSONObject) {
JSONObject obj = (JSONObject) mst;
if (null != obj) {
List<QueryBuilder> qb = getQueryBuilderFromJSON(obj);
if (null != qb) {
pqb.must(qb.get(0));
}
}
} else if (mst instanceof JSONArray) {
JSONArray array = (JSONArray) mst;
JSONObject[] objs = new JSONObject[array.size()];
array.toArray(objs);
for (JSONObject obj : objs) {
List<QueryBuilder> qb = getQueryBuilderFromJSON(obj);
if (null != qb) {
pqb.must(qb.get(0));
}
}
}
pqbs.add(pqb);
Object mstNot = bool.get("must_not");
if (mstNot instanceof JSONObject) {
JSONObject obj = (JSONObject) mstNot;
if (null != obj) {
List<QueryBuilder> qb = getQueryBuilderFromJSON(obj);
if (null != qb) {
pqb.mustNot(qb.get(0));
}
}
} else if (mstNot instanceof JSONArray) {
JSONArray array = (JSONArray) mstNot;
JSONObject[] objs = new JSONObject[array.size()];
array.toArray(objs);
for (JSONObject obj : objs) {
if (null != obj) {
List<QueryBuilder> qb = getQueryBuilderFromJSON(obj);
if (null != qb) {
pqb.mustNot(qb.get(0));
}
}
}
}
pqbs.add(pqb);
Object should = bool.get("should");
if (should instanceof JSONObject) {
JSONObject obj = (JSONObject) should;
if (null != obj) {
List<QueryBuilder> qb = getQueryBuilderFromJSON(obj);
if (null != qb) {
pqb.mustNot(qb.get(0));
}
}
} else if (should instanceof JSONArray) {
JSONArray array = (JSONArray) should;
JSONObject[] objs = new JSONObject[array.size()];
array.toArray(objs);
for (JSONObject obj : objs) {
if (null != obj) {
List<QueryBuilder> qb = getQueryBuilderFromJSON(obj);
if (null != qb) {
pqb.mustNot(qb.get(0));
}
}
}
}
pqbs.add(pqb);
return pqbs;
}
private static class AnalyzeResult {
private int slop = 0;
private List<String> list;
public AnalyzeResult() {
}
public AnalyzeResult(int slop, List<String> lst) {
this.slop = slop;
this.list = lst;
}
public int getSlop() {
return slop;
}
public void setSlop(int slop) {
this.slop = slop;
}
public List<String> getList() {
return list;
}
public void setList(List<String> list) {
this.list = list;
}
}
private static AnalyzeResult getAnalyzeResult(String termStr) {
if (Strings.isNullOrEmpty(termStr)) {
return null;
}
AnalyzeResult rst = new AnalyzeResult();
List<String> list = new ArrayList<>();
if (termStr.trim().length() == 1) {
list.add(termStr);
rst.setList(list);
rst.setSlop(0);
return rst;
}
ESServerUtils esServerUtil = new ESServerUtils();
// todo
if (null == esServerUtil.getClientByClusterId(1L)) {
esServerUtil.initEsServer();
}
// TODO
IndicesAdminClient adminClient = esServerUtil.getClientByClusterId(1L).admin().indices();
// ESHandler esHandler = ESHandler.getESHandler("bfd_mediaforce", "192.168.189.84:9300");
// IndicesAdminClient adminClient = esHandler.getClient().admin().indices();
//
AnalyzeResponse analyzeResponse = adminClient.prepareAnalyze(termStr).setAnalyzer("ik").execute().actionGet();
List<AnalyzeToken> lst = analyzeResponse.getTokens();
// 分词失败
if (null == lst || lst.isEmpty()) {
list.add(termStr);
rst.setList(list);
rst.setSlop(0);
return rst;
} else {
HashSet<AnalyzeToken> exclude = new HashSet<>();
for (AnalyzeToken tk : lst) {
for (AnalyzeToken at : lst) {
if (!tk.getTerm().equals(at.getTerm()) && at.getTerm().contains(tk.getTerm())) {
exclude.add(tk);
}
}
}
List<AnalyzeToken> waitLst = new ArrayList<>();
for (AnalyzeToken tk : lst) {
if (!exclude.contains(tk)) {
waitLst.add(tk);
}
}
if (waitLst.size() == 1) {
list.add(waitLst.get(0).getTerm());
rst.setList(list);
rst.setSlop(0);
return rst;
}
// multi-term
int maxDistance = 0;
for (int i = 0; i < waitLst.size(); i++) {
AnalyzeToken tk = waitLst.get(i);
list.add(tk.getTerm());
if (i > 0) {
AnalyzeToken prv = waitLst.get(i - 1);
int dis = tk.getPosition() - prv.getPosition() - 1;
if (dis > maxDistance) {
maxDistance = dis;
}
}
}
rst.setList(list);
rst.setSlop(maxDistance);
}
// list = dropShorterStr(list);
return rst;
}
private static List<String> dropShorterStr(List<String> lst) {
if (null == lst || lst.isEmpty()) {
return null;
}
HashSet<String> set = new HashSet<>();// 要排除的term
for (String st : lst) {
// 删除较短的分词term
for (String s : lst) {
if (!st.equals(s) && s.contains(st)) {
// lst.remove(st);
set.add(st);
}
}
}
List<String> rst = new ArrayList<>();
for (String st0 : lst) {
if (!set.contains(st0)) {
rst.add(st0);
}
}
return rst;
}
private static boolean listContanins(List<String> lst, String str) {
if (null == lst || lst.isEmpty()) {
return false;
}
for (String st : lst) {
if (st.equals(str)) {
return true;
}
}
return false;
}
}

110
cl_search_api/src/main/java/com/bfd/mf/common/util/thread/SubjectDataQueryThread.java
View File

@ -1,110 +0,0 @@
package com.bfd.mf.common.util.thread;
import com.bfd.mf.common.util.constants.ESConstant;
import net.logstash.logback.encoder.org.apache.commons.lang.exception.ExceptionUtils;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.search.slice.SliceBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import java.util.Random;
import java.util.concurrent.CountDownLatch;
public class SubjectDataQueryThread implements Runnable {
private static final Logger logger = LoggerFactory.getLogger(SubjectDataQueryThread.class);
private static Random random = new Random(System.currentTimeMillis());
private final List<SearchResponse> searchResponseList;
private SearchRequestBuilder searchRequestBuilder;
private TransportClient transportClient;
private final CountDownLatch latch;
public SubjectDataQueryThread(final List<SearchResponse> searchResponseList,
SearchRequestBuilder searchRequestBuilder,
TransportClient transportClient,
CountDownLatch latch) {
this.searchResponseList = searchResponseList;
this.searchRequestBuilder = searchRequestBuilder;
this.transportClient = transportClient;
this.latch = latch;
}
@Override
public void run() {
try {
Thread.sleep(random.nextInt(100));
Integer sliceMax = 3;
// 启用slice scroll
for (int j = 0; j < sliceMax; j++) {
Long start = System.currentTimeMillis();
SliceBuilder sliceBuilder = new SliceBuilder(ESConstant.PUBTIME, j, sliceMax);
//游标分页
searchRequestBuilder.setSize(ESConstant.MAX_SCROLL_SIZE) // 5000
.setScroll(TimeValue.timeValueMinutes(ESConstant.MAX_SEARCH_TIME_MINUTES)) //2L
.slice(sliceBuilder);
SearchResponse searchResponse = searchRequestBuilder.get();
searchResponseList.add(searchResponse);
String scrollId = searchResponse.getScrollId();
int currentResultsNumber = searchResponse.getHits().getHits().length;
while (currentResultsNumber == ESConstant.MAX_SCROLL_SIZE) { // 5000
SearchResponse searchResponsePage = transportClient
.prepareSearchScroll("inner_scroll" + scrollId)
.setScrollId(scrollId)
.setScroll(TimeValue.timeValueMinutes(ESConstant.MAX_SEARCH_TIME_MINUTES)) // 2L
.get();
scrollId = searchResponsePage.getScrollId();
currentResultsNumber = searchResponsePage.getHits().getHits().length;
searchResponseList.add(searchResponsePage);
}
logger.info("SubjectDataQueryThread used time {}" ,System.currentTimeMillis()-start);
}
latch.countDown();
} catch (Exception e) {
logger.error("当前线程errors = {},线程 = {} ", ExceptionUtils.getFullStackTrace(e), Thread.currentThread().getId());
}
}
// @Override
// public void run() {
// try {
// Thread.sleep(random.nextInt(100));
// Integer sliceMax = 3;
// // 启用slice scroll
// for (int j = 0; j < sliceMax; j++) {
// Long start = System.currentTimeMillis();
// SliceBuilder sliceBuilder = new SliceBuilder(ESConstant.PUBTIME, j, sliceMax);
// //游标分页
// searchRequestBuilder.setSize(ESConstant.MAX_SCROLL_SIZE) // 5000
// .setScroll(TimeValue.timeValueMinutes(ESConstant.MAX_SEARCH_TIME_MINUTES)) //2L
// .slice(sliceBuilder);
// SearchResponse searchResponse = searchRequestBuilder.get();
// searchResponseList.add(searchResponse);
// String scrollId = searchResponse.getScrollId();
// int currentResultsNumber = searchResponse.getHits().getHits().length;
// while (currentResultsNumber == ESConstant.MAX_SCROLL_SIZE) { // 5000
// System.out.println(" 进入 while 循环的次数是多少呀？？？");
// SearchResponse searchResponsePage = transportClient
// .prepareSearchScroll("inner_scroll" + scrollId)
// .setScrollId(scrollId)
// .setScroll(TimeValue.timeValueMinutes(ESConstant.MAX_SEARCH_TIME_MINUTES)) // 2L
// .get();
// scrollId = searchResponsePage.getScrollId();
// currentResultsNumber = searchResponsePage.getHits().getHits().length;
// searchResponseList.add(searchResponsePage);
// }
// logger.info("SubjectDataQueryThread used time {}" ,System.currentTimeMillis()-start);
// }
// latch.countDown();
// } catch (Exception e) {
// logger.error("当前线程errors = {},线程 = {} ", ExceptionUtils.getFullStackTrace(e), Thread.currentThread().getId());
// }
// }
}

123
cl_search_api/src/main/java/com/bfd/mf/common/util/utility/CollectionUtils.java
View File

@ -1,123 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.util.utility;
import com.alibaba.fastjson.JSONObject;
import java.util.*;
import java.util.concurrent.CopyOnWriteArrayList;
public class CollectionUtils {
public static Map sortByValue(Map map,int topSize) {
List list = new LinkedList(map.entrySet());
Collections.sort(list, new Comparator() {
public int compare(Object o1, Object o2) {
return -((Comparable) ((Map.Entry) (o1)).getValue())
.compareTo(((Map.Entry) (o2)).getValue());
}
});
Map result = new LinkedHashMap();
int index = 0;
for (Iterator it = list.iterator(); it.hasNext();) {
Map.Entry entry = (Map.Entry) it.next();
index++;
result.put(entry.getKey(), entry.getValue());
if(index == topSize){
break;
}
}
return result;
}
public static List<JSONObject> sortByValueForList(Map map, int topSize) {
List list = new LinkedList(map.entrySet());
Collections.sort(list, new Comparator() {
@Override
public int compare(Object o1, Object o2) {
return -((Comparable) ((Map.Entry) (o1)).getValue())
.compareTo(((Map.Entry) (o2)).getValue());
}
});
List<JSONObject> listResult = new CopyOnWriteArrayList<>();
int index = 0;
for (Iterator it = list.iterator(); it.hasNext();) {
Map.Entry entry = (Map.Entry) it.next();
JSONObject json= new JSONObject();
json.put("key",entry.getKey());
json.put("value",entry.getValue());
listResult.add(json);
index++;
if(index == topSize){
break;
}
}
return listResult;
}
public static List<JSONObject> sortByValueForListSubTopSize(Map map, int topSize, Map resultMap) {
//Map<String,Integer> subMap = new HashMap<>();
List list = new LinkedList(map.entrySet());
Collections.sort(list, new Comparator() {
@Override
public int compare(Object o1, Object o2) {
return -((Comparable) ((Map.Entry) (o1)).getValue())
.compareTo(((Map.Entry) (o2)).getValue());
}
});
List<JSONObject> listResult = new CopyOnWriteArrayList<>();
int index = 0;
for (Iterator it = list.iterator(); it.hasNext();) {
Map.Entry entry = (Map.Entry) it.next();
JSONObject json= new JSONObject();
//json.put("key",entry.getKey());
//json.put("value",entry.getValue());
resultMap.put(entry.getKey(),entry.getValue());
//listResult.add(json);
index++;
if(index == topSize){
break;
}
}
return listResult;
}
// public static void main(String[] args) {
// String aaa = " {\"a1@呲:\":1,\"台湾\":3,\"非洲\":1,\"北:\":1,\" 除非\":1,\"阿卷\":1,\"啼啼\":1,\"帝都\":1,\"//@佐佐山\":3,\"明城\":1,\"上海人民公园\":9,\" //@林\":1,\"黄菡\":4,\"://@佐佐山\":3,\"//@曹\":1,\"阿梳卡:\":1,\"西索\":2,\"景.\":1,\"@Taru-欣\":1,\"旁友圈\":1,\"罗诘:\":1,\"东京\":2,\"似乎\":1,\"阿门\":1,\"丹-\":1,\"@江\":1,\"美国\":5,\"南山 \":1,\"小南\":1,\"//@尒\":1,\"上海\":1,\"香港\":1,\"-阳\":1,\"吧\":2,\"嘛非\":1,\"找华哥\":2,\"//@丸山\":1,\"//@明海\":1,\"滚南\":1,\"长久\":1,\"兄陵\":1,\"南昌路\":1,\"@\":1,\"://@九鹭非\":1,\"压力山\":1,\"环岛\":2,\"啊\":1,\"～干\":1,\"大同\":1,\"乎\":1,\" //@\":1,\"魏洲\":1,\"亚洲\":1,\" //@佐佐山\":1,\"香:\":1,\"宁愿单\":1,\"瑞典\":1,\"//@日村\":1,\"江南\":2,\"死//@\":1,\" //@坂\":1,\"江苏\":1,\"//@\":1,\"@秀山\":1,\"伦比\":1,\"焦虑\":1,\"啦\":1,\"大野蘭\":1,\"微博\":1,\" //@兰\":3,\"越久\":1,\"日本\":17,\"//@九鹭非\":1,\"//@孙\":1,\"南山\":1,\" //@番\":1,\"//@坂\":1,\"浸山\":1,\"//@切尔西\":1,\"二师\":1,\"中國\":3,\"阿瓦卡\":2,\"po//@安\":1,\"乎~\":1,\"新加坡\":1,\"中国\":62,\"台独港\":1,\"//@明河\":1,\"//@路路\":2,\"://@江\":2,\"攻城\":1}";
// Map<String, Integer> map = (Map<String, Integer>)JSONObject.parse(aaa);
// Map<String, Integer> resultMap = new HashMap<String, Integer>();
// String E1 = "[`~!@$%^&*()+=|{}':;',.<>/?~！@￥%……&*（）——+|{}【】‘；：”“’。，、？]";
// String E2 = "[a-zA-Z]";// 英文
// String E3 = "[0-9]";// 数字
// String values = "";
// for (Map.Entry<String, Integer> entry : map.entrySet()) {
// values = entry.getKey().replaceAll(E1, "");
// values = values.replaceAll(E2, "");
// values = values.replaceAll(E3, "");
// resultMap.put(values, entry.getValue());
// System.out.println(values);
// }
// }
}

292
cl_search_api/src/main/java/com/bfd/mf/common/util/utility/DateUtil.java
View File

@ -1,292 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.util.utility;
import com.bfd.nlp.common.util.string.TStringUtils;
import java.io.UnsupportedEncodingException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
public class DateUtil {
private static final String DATE_UNIT_DAY = "D";
public static final String DATE_UNIT_HOUR = "H";
public static String TIME_FORMAT = "yyyy-MM-dd HH:mm:ss";
public static String DATE_FORMAT = "yyyy-MM-dd";
/**
* @param startTime 开始时间
* @param endTime 结束时间
* @param unit D H
* @return
*/
public static double getTimeIntervalByUnit(long startTime, long endTime, String unit) {
int interval = 0;
long dateDistance = endTime - startTime;
if (null == unit || dateDistance <= 0)
return -1;
if (DATE_UNIT_DAY.equals(unit))
interval = 24 * 3600 * 1000;
if (DATE_UNIT_HOUR.equals(unit))
interval = 3600 * 1000;
return Math.ceil(dateDistance / interval);
}
/*
* 20160807190815678:yyyyMMddhhmmssSSS
*/
public static String getTimeStrForNow() {
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmssSSS");
return sdf.format(new Date());
}
private static String getTimeStrDefault() {
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmssSSS");
Date date = new Date();
date.setYear(1970);
return sdf.format(date);
}
public static byte[] timeStr2Chars(String timeStr) {
try {
return timeStr.getBytes("UTF-8");
} catch (UnsupportedEncodingException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return new byte[0];
}
public static long timeStr2Long(String timeStr) {
if (TStringUtils.isEmpty(timeStr)) {
String defTm = getTimeStrDefault();
return Long.parseLong(defTm);
}
return Long.parseLong(timeStr);
}
private static Date parseDate(long time) {
return new Date(time);
}
/***
* timestamp to yyyy-MM-dd
*
* @param timestamp
* @return
*/
public static String parseDateByday(long timestamp) {
Date date = parseDate(timestamp);
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
return format.format(date);
}
/***
* timestamp to yyyy-MM-dd HH:mm:ss
*
* @param timestamp
* @return
*/
public static String parseDateByTime(long timestamp) {
Date date = parseDate(timestamp);
SimpleDateFormat format = new SimpleDateFormat(TIME_FORMAT);
return format.format(date);
}
/**
* timestamp with special format
*
* @param timestamp
* @param format
* @return
*/
public static String parseDateByFormat(long timestamp, String format) {
Date date = parseDate(timestamp);
SimpleDateFormat dateFormat = new SimpleDateFormat(format);
return dateFormat.format(date);
}
/**
* 获取今天是周几
*
* @return 一个表示周几的数字
*/
public static int getDay() {
Calendar cal = Calendar.getInstance();
int day = cal.get(Calendar.DAY_OF_WEEK) - 1;
day = day == 0 ? 7 : day;
return day;
}
/**
* 获取现在是今天的多少秒
*
* @return 一个数字表示现在是今天的多少秒
*/
public static int getSecondsNow() {
Calendar curDate = Calendar.getInstance();
Calendar tommorowDate = new GregorianCalendar(curDate
.get(Calendar.YEAR), curDate.get(Calendar.MONTH), curDate
.get(Calendar.DATE) + 1, 0, 0, 0);
return 24 * 3600 - ((int) (tommorowDate.getTimeInMillis() - curDate.getTimeInMillis()) / 1000);
}
public static class CronDate extends Date {
private int hour;
private int minute;
public CronDate(int h, int m) {
this.hour = h;
this.minute = m;
}
CronDate() {
this.hour = 0;
this.minute = 0;
}
int getHour() {
return hour;
}
void setHour(int hour) {
this.hour = hour;
}
public int getMinute() {
return minute;
}
public void setMinute(int minute) {
this.minute = minute;
}
public boolean before(CronDate date) {
if (null == date) {
return false;
}
if (date.getHour() != this.getHour()) {
return (this.getHour() - date.getHour() < 0);
}
// compare minute
return (this.getMinute() - date.getMinute() < 0);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
if (hour >= 10) {
sb.append(hour);
} else {
sb.append("0").append(hour);
}
sb.append(":");
if (minute >= 10) {
sb.append(minute);
} else {
sb.append("0").append(minute);
}
return sb.toString();
}
// @Override
// public String toString() {
// Date date = new Date();
// date.setHours(hour);
// date.setMinutes(minute);
// String str = cronDdateFormate.format(date);
// return str;
// }
}
/**
* @param dateStr
* @return
*/
public static CronDate parseDateFromStr(String dateStr) {
if (TStringUtils.isEmpty(dateStr)) {
return null;
}
String[] ts = dateStr.split(":");
if (null == ts || ts.length == 0) {
return null;
}
CronDate date = new CronDate();
for (int i = 0; i < ts.length; i++) {
String s = ts[i];
int num = parseDoubleStr(s);
if (i == 0) {
date.setHour(num);
} else if (i == 1) {
date.setMinute(num);
}
}
return date;
}
/**
* @param st
* @return
*/
private static Integer parseDoubleStr(String st) {
if (TStringUtils.isEmpty(st)) {
return null;
}
while (st.startsWith("0") && st.length() > 1) {
st = st.substring(1);
}
if (TStringUtils.isEmpty(st)) {
return 0;
}
return Integer.parseInt(st);
}
/**
* 获取当前时间的小时数和分钟数
*
* @return
*/
public static int[] getCurrentHourAndMinute() {
int[] dat = new int[2];
Date date = new Date();
dat[0] = date.getHours();
dat[1] = date.getMinutes();
return dat;
}
// public static String extractDataScope(long from, long to, boolean fileName) {
// return fileName ?
// (MfTimeUtil.getCSTDateStr(from, "yyyyMMdd") + "_"
// + MfTimeUtil.getCSTDateStr(to, "yyyyMMdd"))
// : ("[" + MfTimeUtil.getCSTDateStr(from, "yyyy-MM-dd") + " ~ "
// + MfTimeUtil.getCSTDateStr(to, "yyyy-MM-dd") + "]");
//
// }
public static Date stringToDate(String dateStr) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
try {
return sdf.parse(dateStr);
} catch (ParseException e) {
return new Date();
}
}
}

18
cl_search_api/src/main/java/com/bfd/mf/common/util/utility/EsQueryConditionUtils.java
View File

@ -1,18 +0,0 @@
package com.bfd.mf.common.util.utility;
public class EsQueryConditionUtils {
public static String[] getIncludeQueryField(String... includeField) {
String[] includeFields = new String[includeField.length];
System.arraycopy(includeField, 0, includeFields, 0, includeField.length - 1 + 1);
return includeFields;
}
public static String[] getExcludeQueryField(String... excludeField) {
String[] excludeFields = new String[excludeField.length];
System.arraycopy(excludeField, 0, excludeFields, 0, excludeField.length - 1 + 1);
return excludeFields;
}
}

518
cl_search_api/src/main/java/com/bfd/mf/common/util/utility/TagWordUtils.java
View File

@ -1,518 +0,0 @@
package com.bfd.mf.common.util.utility;
import org.apache.commons.collections.CollectionUtils;
import java.lang.reflect.Array;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
public class TagWordUtils {
private String tagBegin;
private String tagEnd;
Branch frontbegin = null;
Set<String> keyWords = new HashSet<>();
public Set<String> getKeyWords() {
return keyWords;
}
public String[] getKeyWordsArray() {
if (CollectionUtils.isNotEmpty(keyWords)) {
String[] keyWordsArray = new String[keyWords.size()];
return keyWords.toArray(keyWordsArray);
}
return null;
}
public TagWordUtils(String begin, String end) {
this.tagBegin = begin;
this.tagEnd = end;
}
public TagWordUtils addKeyWords(String[] keyWord) {
if (keyWord.length > 0) {
for (int i = 0; i < keyWord.length; i++) {
this.keyWords.add(keyWord[i].trim());
}
}
return this;
}
// 是否发现词
boolean findWord = false;
/**
* Eric Added 2017-05-15 19:28:43
* If keep break lines \n in content
*
* @param content
* @param keepBr
* @return
*/
public String getTagContentWithBr(String content, boolean keepBr) {
if (content == null || content.trim().length() == 0
|| keyWords.size() == 0) {
return content;
}
// 对原始高亮的信息，进行去掉标签的操作;并且进行去掉 \n \r 以及特殊字符
content = content.replaceAll(" ", "").replaceAll("　", "").replaceAll("\t", "").replaceAll("&nbsp;", "")
.replaceAll("　　", "");
if (!keepBr) {
content = content.replaceAll("\r", "").replaceAll("\n", "");
}
this.frontbegin = new MakeLibrary().getStringTree(this.keyWords);
if (frontbegin == null) {
return content;
}
char[] chars = content.toCharArray();
// 正文
StringBuilder sb = new StringBuilder();
WoodInterface head = this.frontbegin;
int start = 0;
int end = 1;
int index = 0;
boolean isBack = false;
int length = chars.length;
// 此处是正向最大匹配
for (int i = 0; i < length; i++) {
index++;
head = head.get(Character.toLowerCase(chars[i]));
if (head == null) {
if (isBack) {
sb.append(tagBegin).append(chars, start, end).append(tagEnd);
start = start + end;
i = start - 1;
isBack = false;
} else {
sb.append(chars, start, end);
i = start;
start++;
}
head = this.frontbegin;
index = 0;
end = 1;
continue;
}
switch (head.getStatus()) {
case 1:
break;
case 2:
end = index;
isBack = true;
break;
case 3:
sb.append(tagBegin).append(chars, start, index).append(tagEnd);
start = start + index;
index = 0;
end = 1;
isBack = false;
head = this.frontbegin;
break;
}
}
return sb.toString();
}
/**
* 外媒格式处理
*
* @param content
* @return
*/
public String getForeignTagContent(String content) {
if (content == null || content.trim().length() == 0
|| keyWords.size() == 0) {
return content;
}
this.frontbegin = new MakeLibrary().getStringTree(this.keyWords);
if (frontbegin == null) {
return content;
}
char[] chars = content.toCharArray();
// 正文
StringBuilder sb = new StringBuilder();
WoodInterface head = this.frontbegin;
int start = 0;
int end = 1;
int index = 0;
boolean isBack = false;
int length = chars.length;
// 此处是正向最大匹配
for (int i = 0; i < length; i++) {
index++;
head = head.get(Character.toLowerCase(chars[i]));
if (head == null) {
if (isBack) {
sb.append(tagBegin).append(chars, start, end).append(tagEnd);
start = start + end;
i = start - 1;
isBack = false;
} else {
sb.append(chars, start, end);
i = start;
start++;
}
head = this.frontbegin;
index = 0;
end = 1;
continue;
}
switch (head.getStatus()) {
case 1:
break;
case 2:
end = index;
isBack = true;
break;
case 3:
sb.append(tagBegin).append(chars, start, index).append(tagEnd);
start = start + index;
index = 0;
end = 1;
isBack = false;
head = this.frontbegin;
break;
}
}
return sb.toString();
}
/**
* Get content without break lines
*
* @param content
* @return
*/
public String getTagContent(String content) {
return getTagContentWithBr(content, false);
}
// public static void main(String[] args) {
// String[] keyWords = {"智能POS"};
// for (int i = 0; i < 1; i++) {
// String str = new TagWordUtils("<em>", "</em>").addKeyWords(keyWords)
// .getTagContent("智能pos真的好");
// System.out.println(str);
// }
// }
}
class MakeLibrary {
public MakeLibrary() {
}
// 是否有下一个
private static boolean hasNext = true;
// 是否是一个词
private static boolean isWords = true;
Iterator<String> it = null;
public Branch getStringTree(Set<String> keyWords) {
it = keyWords.iterator();
Branch head = new Branch('h', 0, 0);
Branch branch = head;
while (it.hasNext()) {
/**
* 对于英文字母全部都全部转换成小写
*/
char[] chars = it.next().toLowerCase().toCharArray();
for (int i = 0; i < chars.length; i++) {
if (chars.length == (i + 1)) {
isWords = true;
hasNext = false;
} else {
isWords = false;
hasNext = true;
}
int status = 1;
if (isWords && hasNext) {
status = 2;
}
if (!isWords && hasNext) {
status = 1;
}
if (isWords && !hasNext) {
status = 3;
}
branch.add(new Branch(chars[i], status, 0));
branch = (Branch) branch.get(chars[i]);
}
branch = head;
}
return head;
}
}
interface WoodInterface {
public WoodInterface add(WoodInterface branch);
public WoodInterface get(char c);
public boolean contains(char c);
public int compareTo(char c);
public boolean equals(char c);
public byte getStatus();
public char getC();
public void setStatus(int status);
public byte getNature();
public void setNature(byte nature);
}
class Branch implements WoodInterface {
/**
* status 此字的状态1，继续 2，是个词语但是还可以继续 ,3确定
* nature 词语性质
* 0.未知 . 1是姓 . 2 是职位名称 3 是数量级的词 . 4 是数字词语 5 是标点
*/
WoodInterface[] branches = new WoodInterface[0];
private char c;
// 状态
private byte status = 1;
// 索引
private short index = -1;
// 词性
private byte nature = 0;
// 单独查找出来的对象
WoodInterface branch = null;
public WoodInterface add(WoodInterface branch) {
if ((this.branch = this.get(branch.getC())) != null) {
switch (branch.getStatus()) {
case 1:
if (this.branch.getStatus() == 2) {
this.branch.setStatus(2);
}
if (this.branch.getStatus() == 3) {
this.branch.setStatus(2);
}
break;
case 2:
this.branch.setStatus(2);
case 3:
if (this.branch.getStatus() == 2) {
this.branch.setStatus(2);
}
if (this.branch.getStatus() == 1) {
this.branch.setStatus(2);
}
}
this.branch.setNature(branch.getNature());
return this.branch;
}
index++;
if ((index + 1) > branches.length) {
branches = Arrays.copyOf(branches, index + 1);
}
branches[index] = branch;
AnsjArrays.sort(branches);
return branch;
}
public Branch(char c, int status, int nature) {
this.c = c;
this.status = (byte) status;
this.nature = (byte) nature;
}
int i = 0;
public WoodInterface get(char c) {
int i = AnsjArrays.binarySearch(branches, c);
if (i > -1) {
return branches[i];
}
return null;
}
public boolean contains(char c) {
if (AnsjArrays.binarySearch(branches, c) > -1) {
return true;
} else {
return false;
}
}
public int compareTo(char c) {
if (this.c > c) {
return 1;
} else if (this.c < c) {
return -1;
} else {
return 0;
}
}
public boolean equals(char c) {
if (this.c == c) {
return true;
} else {
return false;
}
}
@Override
public int hashCode() {
// TODO Auto-generated method stub
return c;
}
public byte getStatus() {
return status;
}
public void setStatus(int status) {
this.status = (byte) status;
}
public char getC() {
return this.c;
}
public byte getNature() {
return nature;
}
public void setNature(byte nature) {
this.nature = nature;
}
}
class AnsjArrays {
private static final int INSERTIONSORT_THRESHOLD = 7;
/**
* 二分法查找.摘抄了jdk的东西..只不过把他的自动装箱功能给去掉了
*/
public static int binarySearch(WoodInterface[] branches, char c) {
int high = branches.length - 1;
if (branches.length < 1) {
return high;
}
int low = 0;
while (low <= high) {
int mid = (low + high) >>> 1;
int cmp = branches[mid].compareTo(c);
if (cmp < 0) {
low = mid + 1;
} else if (cmp > 0) {
high = mid - 1;
} else {
return mid; // key found
}
}
return -1; // key not found.
}
public static void sort(WoodInterface[] a) {
WoodInterface[] aux = (WoodInterface[]) a.clone();
mergeSort(aux, a, 0, a.length, 0);
}
public static void sort(WoodInterface[] a, int fromIndex, int toIndex) {
rangeCheck(a.length, fromIndex, toIndex);
WoodInterface[] aux = copyOfRange(a, fromIndex, toIndex);
mergeSort(aux, a, fromIndex, toIndex, -fromIndex);
}
private static void rangeCheck(int arrayLen, int fromIndex, int toIndex) {
if (fromIndex > toIndex) {
throw new IllegalArgumentException("fromIndex(" + fromIndex
+ ") > toIndex(" + toIndex + ")");
}
if (fromIndex < 0) {
throw new ArrayIndexOutOfBoundsException(fromIndex);
}
if (toIndex > arrayLen) {
throw new ArrayIndexOutOfBoundsException(toIndex);
}
}
private static void mergeSort(WoodInterface[] src, WoodInterface[] dest, int low,
int high, int off) {
int length = high - low;
// Insertion sort on smallest arrays
if (length < INSERTIONSORT_THRESHOLD) {
for (int i = low; i < high; i++)
for (int j = i; j > low
&& (dest[j - 1]).compareTo(dest[j].getC()) > 0; j--)
swap(dest, j, j - 1);
return;
}
// Recursively sort halves of dest into src
int destLow = low;
int destHigh = high;
low += off;
high += off;
int mid = (low + high) >>> 1;
mergeSort(dest, src, low, mid, -off);
mergeSort(dest, src, mid, high, -off);
// If list is already sorted, just copy from src to dest. This is an
// optimization that results in faster sorts for nearly ordered lists.
if (src[mid - 1].compareTo(src[mid].getC()) <= 0) {
System.arraycopy(src, low, dest, destLow, length);
return;
}
// Merge sorted halves (now in src) into dest
for (int i = destLow, p = low, q = mid; i < destHigh; i++) {
if (q >= high || p < mid
&& src[p].compareTo(src[q].getC()) <= 0) {
dest[i] = src[p++];
} else {
dest[i] = src[q++];
}
}
}
/**
* Swaps x[a] with x[b].
*/
private static void swap(WoodInterface[] x, int a, int b) {
WoodInterface t = x[a];
x[a] = x[b];
x[b] = t;
}
public static <T> T[] copyOfRange(T[] original, int from, int to) {
return copyOfRange(original, from, to, (Class<T[]>) original.getClass());
}
public static <T, U> T[] copyOfRange(U[] original, int from, int to,
Class<? extends T[]> newType) {
int newLength = to - from;
if (newLength < 0) {
throw new IllegalArgumentException(from + " > " + to);
}
T[] copy = ((Object) newType == (Object) Object[].class) ? (T[]) new Object[newLength]
: (T[]) Array
.newInstance(newType.getComponentType(), newLength);
System.arraycopy(original, from, copy, 0, Math.min(original.length
- from, newLength));
return copy;
}
}

72
cl_search_api/src/main/java/com/bfd/mf/common/web/component/wrapper/ResponseWrapper.java
View File

@ -1,72 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.component.wrapper;
import com.alibaba.fastjson.JSONObject;
import com.bfd.mf.common.util.enums.RTCodeEnum;
import com.bfd.nlp.common.util.object.TObjectUtils;
import com.bfd.nlp.common.util.string.TStringUtils;
import com.google.common.base.Strings;
import java.util.HashMap;
import java.util.List;
public class ResponseWrapper {
public static JSONObject buildResponse(RTCodeEnum codeEnum, String message, Object data) {
JSONObject result = new JSONObject(true);
result.put("code", codeEnum.getCode());
if (!TStringUtils.isEmpty(message)) {
result.put("message", message);
}
result.put("desc", codeEnum.getDesc());
if (!TObjectUtils.isNull(data)) {
result.put("data", data);
}
return result;
}
public static JSONObject buildResponse(RTCodeEnum codeEnum, Object data) {
return buildResponse(codeEnum, null, data);
}
public static JSONObject buildResponse(RTCodeEnum codeEnum, String message) {
return buildResponse(codeEnum, message, null);
}
public static JSONObject buildResponse(RTCodeEnum codeEnum) {
return buildResponse(codeEnum, null, null);
}
public static HashMap<Long, List<JSONObject>> buildHashResponse(RTCodeEnum codeEnum, String message) {
HashMap<Long, List<JSONObject>> hashMapResponse = new HashMap<>(1);
hashMapResponse.put((long) codeEnum.getCode(), null);
return hashMapResponse;
}
public static JSONObject buildResponseWithKey(RTCodeEnum codeEnum, String message, Object data, String key) {
JSONObject result = new JSONObject(true);
result.put("code", codeEnum.getCode());
if (!TStringUtils.isEmpty(message)) {
result.put("msg", message);
}
if (!TObjectUtils.isNull(data) && !Strings.isNullOrEmpty(key)) {
result.put(key, data);
}
return result;
}
}

41
cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/BookMark.java
View File

@ -1,41 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.entity.mysql;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import java.io.Serializable;
@Entity
@Table(name = "cl_book_mark")
public class BookMark extends Model implements Serializable {
private static final long serialVersionUID = -2077762992475568684L;
@Column(name = "book_mark")
private String markName;
public String getMarkName() {
return markName;
}
public void setMarkName(String markName) {
this.markName = markName;
}
}

64
cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/Channel.java
View File

@ -1,64 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.entity.mysql;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import javax.persistence.Transient;
import java.io.Serializable;
@Entity
@Table(name = "cl_channel")
public class Channel extends Model implements Serializable {
private static final long serialVersionUID = -4246312514918925209L;
@Column(name = "name")
private String name;
@Column(name = "code")
private String code;
@Transient
private Boolean status = false;
public Boolean getStatus() {
return status;
}
public void setStatus(Boolean status) {
this.status = status;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getCode() {
return code;
}
public void setCode(String code) {
this.code = code;
}
}

50
cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/Model.java
View File

@ -1,50 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.entity.mysql;
import org.apache.commons.lang3.builder.ToStringBuilder;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.MappedSuperclass;
import java.io.Serializable;
@MappedSuperclass
public abstract class Model implements Serializable {
private static final long serialVersionUID = 2792083395861331755L;
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
protected Long id;
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
@Override
public String toString() {
return new ToStringBuilder(this)
.append("id", id)
.toString();
}
}

127
cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/SentimentModify.java
View File

@ -1,127 +0,0 @@
package com.bfd.mf.common.web.entity.mysql;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import java.io.Serializable;
import java.util.Date;
@Entity
@Table(name = "cl_sentiment")
public class SentimentModify extends Model implements Serializable {
private static final long serialVersionUID = 8927418266148468982L;
@Column(name = "token")
private String token;
@Column(name = "doc_id")
private String docId;
@Column(name = "doc_url")
private String url;
@Column(name = "sim_hash")
private String simHash;
@Column(name = "doc_type")
private String docType;
@Column(name = "originsentiment")
private String originSentiment;
@Column(name = "newsentiment")
private String newSentiment;
@Column(name = "update_time")
private String updateTime = new Date().toLocaleString();
@Column(name = "doc_title")
private String title;
@Column(name = "content_abs")
private String contentAbstract;
@Column(name = "type")
private Integer type;
public String getToken() {
return token;
}
public void setToken(String token) {
this.token = token;
}
public String getDocId() {
return docId;
}
public void setDocId(String docId) {
this.docId = docId;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public String getSimHash() {
return simHash;
}
public void setSimHash(String simHash) {
this.simHash = simHash;
}
public String getDocType() {
return docType;
}
public void setDocType(String docType) {
this.docType = docType;
}
public String getOriginSentiment() {
return originSentiment;
}
public void setOriginSentiment(String originSentiment) {
this.originSentiment = originSentiment;
}
public String getNewSentiment() {
return newSentiment;
}
public void setNewSentiment(String newSentiment) {
this.newSentiment = newSentiment;
}
public String getUpdateTime() {
return updateTime;
}
public void setUpdateTime(String updateTime) {
this.updateTime = updateTime;
}
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public String getContentAbstract() {
return contentAbstract;
}
public void setContentAbstract(String contentAbstract) {
this.contentAbstract = contentAbstract;
}
public Integer getType() {
return type;
}
public void setType(Integer type) {
this.type = type;
}
}

49
cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/SubjectModel.java
View File

@ -1,49 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.entity.mysql;
import org.apache.commons.lang3.builder.ToStringBuilder;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.MappedSuperclass;
import java.io.Serializable;
import java.math.BigInteger;
@MappedSuperclass
public abstract class SubjectModel implements Serializable {
private static final long serialVersionUID = -1969384421573693998L;
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
protected BigInteger id;
public BigInteger getId() {
return id;
}
public void setId(BigInteger id) {
this.id = id;
}
@Override
public String toString() {
return new ToStringBuilder(this).append("id", id).toString();
}
}

322
cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/User.java
View File

@ -1,322 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.entity.mysql;
import org.apache.commons.lang3.builder.ToStringBuilder;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import javax.persistence.Transient;
import java.io.Serializable;
@Entity
@Table(name = "cl_user")
public class User extends Model implements Serializable {
private static final long serialVersionUID = 1947459646071591960L;
@Column(name = "token")
private String token;
@Column(name = "user_group_id")
private Long userGroupId;
@Column(name = "account")
private String account;
@Column(name = "username")
private String userName;
@Column(name = "password")
private String password;
@Column(name = "parent_id")
private Long parentId;
@Column(name = "email")
private String email;
@Column(name = "wechat")
private String wechat;
@Column(name = "wechat_id")
private String wechatId;
// Y N
@Column(name = "wechat_verified")
private String wechatVerified;
@Column(name = "wechat_open_id")
private String wechatOpenId;
@Column(name = "mobile")
private String mobile;
@Column(name = "role_id")
private String roleId;
@Column(name = "status")
private Integer status;
@Column(name = "last_login_time")
private Long lastLoginTime;
@Column(name = "company_id")
private Long companyId;
@Column(name = "created_time")
private Long createdTime;
@Column(name = "updated_time")
private Long updatedTime;
@Column(name = "del")
private Boolean del;
@Column(name = "is_shadow")
private Integer isShadow;
@Transient
private Integer foreignChannelStatus;
@Transient
private Boolean commentStatus;
@Transient
private Integer cacheStatus;
public Integer getCacheStatus() {
return cacheStatus;
}
public void setCacheStatus(Integer cacheStatus) {
this.cacheStatus = cacheStatus;
}
public User() {
}
public Integer getForeignChannelStatus() {
return foreignChannelStatus;
}
public void setForeignChannelStatus(Integer foreignChannelStatus) {
this.foreignChannelStatus = foreignChannelStatus;
}
public Integer getIsShadow() {
return isShadow;
}
public void setIsShadow(Integer isShadow) {
this.isShadow = isShadow;
}
public Long getUserGroupId() {
return userGroupId;
}
public void setUserGroupId(Long userGroupId) {
this.userGroupId = userGroupId;
}
public String getUserName() {
return userName;
}
public void setUserName(String userName) {
this.userName = userName;
}
public String getToken() {
return token;
}
public void setToken(String token) {
this.token = token;
}
public String getAccount() {
return account;
}
public void setAccount(String account) {
this.account = account;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public long getParentId() {
return parentId;
}
public void setParentId(long parentId) {
this.parentId = parentId;
}
public String getEmail() {
return email;
}
public void setEmail(String email) {
this.email = email;
}
public String getWechat() {
return wechat;
}
public void setWechat(String wechat) {
this.wechat = wechat;
}
public String getWechatId() {
return wechatId;
}
public void setWechatId(String wechatId) {
this.wechatId = wechatId;
}
public String getMobile() {
return mobile;
}
public void setMobile(String mobile) {
this.mobile = mobile;
}
public String getRoleId() {
return roleId;
}
public void setRoleId(String roleId) {
this.roleId = roleId;
}
public void setStatus(Integer status) {
this.status = status;
}
public int getStatus() {
return status;
}
public void setStatus(int status) {
this.status = status;
}
public long getLastLoginTime() {
return lastLoginTime;
}
public Long getCompanyId() {
return companyId;
}
public void setCompanyId(Long companyId) {
this.companyId = companyId;
}
public Long getCreatedTime() {
return createdTime;
}
public void setCreatedTime(Long createdTime) {
this.createdTime = createdTime;
}
public Long getUpdatedTime() {
return updatedTime;
}
public void setUpdatedTime(Long updatedTime) {
this.updatedTime = updatedTime;
}
public void setParentId(Long parentId) {
this.parentId = parentId;
}
public void setLastLoginTime(Long lastLoginTime) {
this.lastLoginTime = lastLoginTime;
}
public Boolean getDel() {
return del;
}
public void setDel(Boolean del) {
this.del = del;
}
public String getWechatVerified() {
return wechatVerified;
}
public void setWechatVerified(String wechatVerified) {
this.wechatVerified = wechatVerified;
}
public String getWechatOpenId() {
return wechatOpenId;
}
public void setWechatOpenId(String wechatOpenId) {
this.wechatOpenId = wechatOpenId;
}
public Boolean getCommentStatus() {
return commentStatus;
}
public void setCommentStatus(Boolean commentStatus) {
this.commentStatus = commentStatus;
}
@Override
public String toString() {
return new ToStringBuilder(this)
.append("token", token)
.append("account", account)
.append("userName", userName)
.append("password", password)
.append("parentId", parentId)
.append("email", email)
.append("wechat", wechat)
.append("wechatId", wechatId)
.append("wechatVerified", wechatVerified)
.append("wechatOpenId", wechatOpenId)
.append("mobile", mobile)
.append("roleId", roleId)
.append("status", status)
.append("lastLoginTime", lastLoginTime)
.append("companyId", companyId)
.append("commentStatus", commentStatus)
.append("createdTime", createdTime)
.append("updatedTime", updatedTime)
.append("del", del)
.toString();
}
}

106
cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/base/BaseEntityModel.java
View File

@ -1,106 +0,0 @@
package com.bfd.mf.common.web.entity.mysql.base;
import com.fasterxml.jackson.annotation.JsonIgnore;
import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
import javax.persistence.*;
import java.util.Date;
@MappedSuperclass
public class BaseEntityModel extends BaseModel {
/**
*
*/
private static final long serialVersionUID = -6163675075289529459L;
/**
* 实体创建时间
*/
@JsonIgnore
@Temporal(TemporalType.TIMESTAMP)
@Column(name = "date_created")
private Date dateCreated = new Date();
/**
* 实体修改时间
*/
@JsonIgnore
@Temporal(TemporalType.TIMESTAMP)
@Column(name = "date_modified")
private Date dateModified = new Date();
/**
* 实体是否被删除
*/
@Column(name = "DELETED")
protected Boolean deleted = false;
/**
* 实体主键
*/
@Id
@Column(name = "ID", nullable = false)
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;
@Override
public boolean equals(Object obj) {
if (null != obj) {
if (obj instanceof BaseEntityModel) {
BaseEntityModel domain = (BaseEntityModel) obj;
if (this.id == domain.id) {
return true;
}
}
}
return false;
}
public Date getDateCreated() {
return dateCreated;
}
public Date getDateModified() {
return dateModified;
}
public Long getId() {
return id;
}
@Override
public int hashCode() {
if (this.id == null) {
this.id = 0L;
}
return HashCodeBuilder.reflectionHashCode(this.id);
}
public Boolean isDeleted() {
return deleted;
}
public void setDateCreated(Date dateCreated) {
this.dateCreated = dateCreated;
}
public void setDateModified(Date dateModified) {
this.dateModified = dateModified;
}
public void setDeleted(Boolean deleted) {
this.deleted = deleted;
}
public void setId(Long id) {
this.id = id;
}
@Override
public String toString() {
return ToStringBuilder.reflectionToString(this,
ToStringStyle.MULTI_LINE_STYLE);
}
}

34
cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/base/BaseModel.java
View File

@ -1,34 +0,0 @@
package com.bfd.mf.common.web.entity.mysql.base;
import java.io.Serializable;
@SuppressWarnings("serial")
public abstract class BaseModel implements Serializable {
/**
* Returns a multi-line String with key=value pairs.
*
* @return a String representation of this class.
*/
public abstract String toString();
/**
* Compares object equality. When using Hibernate, the primary key should
* not be a part of this comparison.
*
* @param o
* object to compare to
* @return true/false based on equality tests
*/
public abstract boolean equals(Object o);
/**
* When you override equals, you should override hashCode. See "Why are
* equals() and hashCode() importation" for more information:
* http://www.hibernate.org/109.html
*
* @return hashCode
*/
public abstract int hashCode();
}

113
cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/base/BaseTreeEntityModel.java
View File

@ -1,113 +0,0 @@
package com.bfd.mf.common.web.entity.mysql.base;
import com.bfd.nlp.common.util.object.TObjectUtils;
import com.fasterxml.jackson.annotation.JsonBackReference;
import com.fasterxml.jackson.annotation.JsonManagedReference;
import javax.persistence.*;
import java.util.Collections;
import java.util.List;
@SuppressWarnings("rawtypes")
@MappedSuperclass
public class BaseTreeEntityModel<T extends BaseTreeEntityModel> extends
BaseEntityModel implements Comparable<T>{
private static final long serialVersionUID = -5961264427451119166L;
@JsonManagedReference("parent-children")
@OneToMany(mappedBy = "parent", cascade = CascadeType.ALL, fetch = FetchType.EAGER)
List<T> children;
@Transient
boolean leaf = true;
@JsonBackReference("parent-children")
@ManyToOne
@JoinColumn(name = "PARENT_ID")
T parent;
@Column(name = "TEXT")
String text;
@Column(name = "sort")
private Integer sort;
public boolean isLeaf() {
if(null != children && children.size() > 0) {
this.leaf = false;
} else {
this.leaf = true;
}
return leaf;
}
public void setLeaf(boolean leaf) {
this.leaf = leaf;
}
public Integer getSort() {
return sort;
}
public void setSort(Integer sort) {
this.sort = sort;
}
public List<T> getChildren() {
if(TObjectUtils.isListEmpty(children)){
return children;
}else{
Collections.sort(children);
return children;
}
}
public T getParent() {
return parent;
}
public String getText() {
return text;
}
public void setChildren(List<T> children) {
this.children = children;
}
public void setParent(T parent) {
this.parent = parent;
}
public void setText(String text) {
this.text = text;
}
/***
* 组内按照 sort 升序排
*/
@Override
public int compareTo(T o) {
if(this.sort > o.getSort())
return 1;
else
return -1;
}
/*@Override
public String toString() {
return ToStringBuilder.reflectionToString(this,
ToStringStyle.MULTI_LINE_STYLE, false);
}*/
/*@Override
public String toString() {
return "BaseTreeEntityModel{" +
"children=" + children +
", leaf=" + leaf +
", birthDate=" + birthDate +
", parent=" + parent +
", text='" + text + '\'' +
'}';
}*/
}

166
cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/cache/Cluster.java
View File

@ -1,166 +0,0 @@
package com.bfd.mf.common.web.entity.mysql.cache;
import com.bfd.mf.common.web.entity.mysql.Model;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import java.io.Serializable;
/**
* Created by Shang Gaohui on 2017/12/26.
*/
@Entity
@Table(name = "cl_cluster")
public class Cluster extends Model implements Serializable, Comparable<Cluster> {
private static final long serialVersionUID = -2048927040596969768L;
public String getEnvironment() {
return environment;
}
public void setEnvironment(String environment) {
this.environment = environment;
}
@Override
public int compareTo(Cluster o) {
if (this.getShardNum() >= o.getShardNum())
return -1;
else
return 1;
}
public static final class CLUSTER_TYPE {
/**
* 小集群
*/
public static final int mini_cluster_type = 2; // 111 的ES 集群 专题索引 SQ_Mini
/**
* 大集群
*/
public static final int normal_cluster_type = 0; // 109 的集群 日期索引 SQ_Normal
// /**
// * 缓存
// */
// // public static final int subject_cluster_type = 2; //
//
// /**
// * 微博转发
// */
// public static final int weibo_cluster_type = 4; // 147的ES 集群 （前台打标后）SQ_Mini
//
// public static final int reply_cluster_type = 7; // 已废弃 （原160的ES）
// /**
// * 2期数据分析 集群
// */
// public static final int data_analysis_cluster_type = 5; // 158 的ES 集群（后台打标后的数据） SQ_Reputation_new
// /**
// * 对应索引以 cl_company_*
// */
// public static final int data_company_analysis_cluster_type = 6; // 已废弃 （原160的ES）
}
public static Long getClusterIdByType(SubjectCluster subjectCluster, int type) {
if (type == Cluster.CLUSTER_TYPE.mini_cluster_type) {
return subjectCluster.getMiniClusterId();
} else if (type == Cluster.CLUSTER_TYPE.normal_cluster_type) {
return subjectCluster.getNormalClusterId();
// } else if (type == Cluster.CLUSTER_TYPE.subject_cluster_type) {
// return subjectCluster.getClusterId();
}
return null;
}
@Column(name = "ip")
private String ip;
@Column(name = "tcp_port")
private String tcpPort;
@Column(name = "http_port")
private String httpPort;
public Long getIndexTemplateId() {
return indexTemplateId;
}
public void setIndexTemplateId(Long indexTemplateId) {
this.indexTemplateId = indexTemplateId;
}
@Column(name = "index_template_id")
private Long indexTemplateId;
public String getPrefixIndexPattern() {
return prefixIndexPattern;
}
public void setPrefixIndexPattern(String prefixIndexPattern) {
this.prefixIndexPattern = prefixIndexPattern;
}
@Column(name = "prefix_index_pattern")
private String prefixIndexPattern;
@Column(name = "cluster_name")
private String clusterName;
@Column(name = "shard_num")
private Integer shardNum;
@Column(name = "type")
private Integer type;
@Column(name = "environment")
private String environment;
public String getIp() {
return ip;
}
public void setIp(String ip) {
this.ip = ip;
}
public String getTcpPort() {
return tcpPort;
}
public void setTcpPort(String tcpPort) {
this.tcpPort = tcpPort;
}
public String getHttpPort() {
return httpPort;
}
public void setHttpPort(String httpPort) {
this.httpPort = httpPort;
}
public String getClusterName() {
return clusterName;
}
public void setClusterName(String clusterName) {
this.clusterName = clusterName;
}
public Integer getShardNum() {
return shardNum;
}
public void setShardNum(Integer shardNum) {
this.shardNum = shardNum;
}
public Integer getType() {
return type;
}
public void setType(Integer type) {
this.type = type;
}
}

114
cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/cache/SubjectCluster.java
View File

@ -1,114 +0,0 @@
package com.bfd.mf.common.web.entity.mysql.cache;
import com.bfd.mf.common.web.entity.mysql.Model;
import com.fasterxml.jackson.annotation.JsonIgnore;
import javax.persistence.*;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.Date;
/**
* Created by Shang Gaohui on 2017/12/26.
*/
@Entity
@Table(name = "cl_subject_cluster")
public class SubjectCluster extends Model implements Serializable {
private static final long serialVersionUID = 4886257870671839342L;
public SubjectCluster(){
}
public Date getDateCreated() {
return dateCreated;
}
public void setDateCreated(Date dateCreated) {
this.dateCreated = dateCreated;
}
/**
* 实体创建时间
* dj - 2020-01-10
*/
@JsonIgnore
@Temporal(TemporalType.TIMESTAMP)
@Column(name = "date_created")
protected Date dateCreated = new Date();
public SubjectCluster(BigInteger subjectId, Long clusterId, Long miniClusterId, Long normalClusterId,String indexName) {
this.subjectId = subjectId;
this.clusterId = clusterId;
this.miniClusterId = miniClusterId;
this.normalClusterId = normalClusterId;
this.indexName = indexName;
}
@Column(name = "subject_id")
private BigInteger subjectId;
@Column(name = "cluster_id")
private Long clusterId;
@Column(name = "index_name")
private String indexName;
@Column(name = "mini_cluster_id")
private Long miniClusterId;
@Column(name = "normal_cluster_id")
private Long normalClusterId;
@Override
public String toString() {
return "SubjectCluster{" +
"dateCreated=" + dateCreated +
", subjectId=" + subjectId +
", clusterId=" + clusterId +
", miniClusterId=" + miniClusterId +
", indexName='" + indexName + '\'' +
", normalClusterId=" + normalClusterId +
'}';
}
public Long getNormalClusterId() {
return normalClusterId;
}
public void setNormalClusterId(Long normalClusterId) {
this.normalClusterId = normalClusterId;
}
public BigInteger getSubjectId() {
return subjectId;
}
public void setSubjectId(BigInteger subjectId) {
this.subjectId = subjectId;
}
public Long getClusterId() {
return clusterId;
}
public void setClusterId(Long clusterId) {
this.clusterId = clusterId;
}
public Long getMiniClusterId() {
return miniClusterId;
}
public void setMiniClusterId(Long miniClusterId) {
this.miniClusterId = miniClusterId;
}
public String getIndexName() {
return indexName;
}
public void setIndexName(String indexName) {
this.indexName = indexName;
}
}

104
cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/report/MainSourceEntity.java
View File

@ -1,104 +0,0 @@
package com.bfd.mf.common.web.entity.mysql.report;
import com.bfd.mf.common.web.entity.mysql.Model;
import lombok.NoArgsConstructor;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
@NoArgsConstructor
@Entity
@Table(name = "cl_main_source")
public class MainSourceEntity extends Model {
public String getDocTypeEn() {
return docTypeEn;
}
public void setDocTypeEn(String docTypeEn) {
this.docTypeEn = docTypeEn;
}
public String getDocTypeCn() {
return docTypeCn;
}
public void setDocTypeCn(String docTypeCn) {
this.docTypeCn = docTypeCn;
}
public String getSource() {
return source;
}
public void setSource(String source) {
this.source = source;
}
public int getStatus() {
return status;
}
public void setStatus(int status) {
this.status = status;
}
public Long getCompanyId() {
return companyId;
}
public void setCompanyId(Long companyId) {
this.companyId = companyId;
}
public Long getLastUpdateTime() {
return lastUpdateTime;
}
public void setLastUpdateTime(Long lastUpdateTime) {
this.lastUpdateTime = lastUpdateTime;
}
public Long getLastUpdateUser() {
return lastUpdateUser;
}
public void setLastUpdateUser(Long lastUpdateUser) {
this.lastUpdateUser = lastUpdateUser;
}
@Column(name = "doc_type_en")
private String docTypeEn;
@Column(name = "doc_type_cn")
private String docTypeCn;
@Column(name = "source")
private String source;
@Column(name = "last_update_time")
private Long lastUpdateTime;
@Column(name = "last_update_user")
private Long lastUpdateUser;
@Column(name = "status")
private int status;
@Column(name = "company_id")
private Long companyId;
@Override
public String toString() {
return "MainSourceEntity{" +
"docTypeEn='" + docTypeEn + '\'' +
", docTypeCn='" + docTypeCn + '\'' +
", source='" + source + '\'' +
", lastUpdateTime='" + lastUpdateTime + '\'' +
", lastUpdateUser='" + lastUpdateUser + '\'' +
", status=" + status +
", companyId=" + companyId +
'}';
}
}

425
cl_search_api/src/main/java/com/bfd/mf/common/web/entity/mysql/topic/Subject.java
View File

@ -1,425 +0,0 @@
package com.bfd.mf.common.web.entity.mysql.topic;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.bfd.mf.common.util.enums.CacheEnums;
import com.bfd.mf.common.web.entity.mysql.SubjectModel;
import com.fasterxml.jackson.annotation.JsonIgnore;
import org.apache.commons.lang3.builder.ToStringBuilder;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import javax.persistence.Transient;
import java.io.Serializable;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
@Entity
@Table(name = "cl_subject")
public class Subject extends SubjectModel implements Serializable {
private static final long serialVersionUID = 1066084624308428301L;
@Column(name = "name")
private String name;
@Column(name = "node_type")
private Integer nodeType;
@Column(name = "keywords_type")
private Integer keywordsType;
@Column(name = "simple_content")
private String simpleContent;
@Column(name = "relation_content")
private String relationContent;
@Column(name = "expert_content")
private String expertContent;
@Column(name = "exclude_content")
private String excludeContent;
@Column(name = "channels")
private String channels;
@Column(name = "create_time")
private Long createTime = System.currentTimeMillis();
@Column(name = "update_time")
private Long updateTime = System.currentTimeMillis();
@Column(name = "kw_expression")
private String kwExpression;
@JsonIgnore
@Column(name = "es_query")
private String esQuery;
@Column(name = "creator")
private String creator;
@Column(name = "company_id")
private Long companyId;
@Column(name = "subject_back_date")
private BigInteger subjectBackDate = BigInteger.valueOf(System.currentTimeMillis());
@Transient
private Long companyBackDate;
@Transient
private boolean isExistWarning = false;
public boolean isExistWarning() {
return isExistWarning;
}
public void setExistWarning(boolean existWarning) {
isExistWarning = existWarning;
}
public Long getCompanyBackDate() {
return companyBackDate;
}
public void setCompanyBackDate(Long companyBackDate) {
this.companyBackDate = companyBackDate;
}
public BigInteger getSubjectBackDate() {
return subjectBackDate;
}
public void setSubjectBackDate(BigInteger subjectBackDate) {
this.subjectBackDate = subjectBackDate;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Integer getNodeType() {
return nodeType;
}
public void setNodeType(Integer nodeType) {
this.nodeType = nodeType;
}
public Integer getKeywordsType() {
return keywordsType;
}
public void setKeywordsType(Integer keywordsType) {
this.keywordsType = keywordsType;
}
public String getSimpleContent() {
return simpleContent;
}
public void setSimpleContent(String simpleContent) {
this.simpleContent = simpleContent;
}
public String getRelationContent() {
return relationContent;
}
public JSONArray getRelationContentArray() {
if (relationContent == null) {
return null;
}
return JSONObject.parseArray(relationContent);
}
public void setRelationContent(String relationContent) {
this.relationContent = relationContent;
}
public String getExpertContent() {
return expertContent;
}
public void setExpertContent(String expertContent) {
this.expertContent = expertContent;
}
public String getCreator() {
return creator;
}
public void setCreator(String creator) {
this.creator = creator;
}
public String getExcludeContent() {
return excludeContent;
}
public void setExcludeContent(String excludeContent) {
this.excludeContent = excludeContent;
}
public String getChannels() {
return channels;
}
public JSONObject getChannelsJson() {
if (channels == null) {
return null;
}
return JSONObject.parseObject(channels);
}
public void setChannels(String channels) {
this.channels = channels;
}
public Long getCreateTime() {
return createTime;
}
public void setCreateTime(Long createTime) {
this.createTime = createTime;
}
public Long getUpdateTime() {
return updateTime;
}
public void setUpdateTime(Long updateTime) {
this.updateTime = updateTime;
}
public String getEsQuery() {
return esQuery;
}
public void setEsQuery(String esQuery) {
this.esQuery = esQuery;
}
public String getKwExpression() {
return kwExpression;
}
public void setKwExpression(String kwExpression) {
this.kwExpression = kwExpression;
}
public Long getCompanyId() {
return companyId;
}
public void setCompanyId(Long companyId) {
this.companyId = companyId;
}
@Transient
private List<Integer> userIdList = new ArrayList<>();
public List<Integer> getUserIdList() {
return userIdList;
}
public void setUserIdList(List<Integer> userIdList) {
this.userIdList = userIdList;
}
@Override
public boolean equals(Object other) {
if (null == other || !(other instanceof Subject)) {
return false;
}
if (this == other) {
return true;
}
BigInteger thisId = this.getId();
BigInteger otherId = ((Subject) other).getId();
if (null != thisId && null != otherId && thisId.longValue() == otherId.longValue()) {
return true;
}
return false;
}
// 缓存相关
@Column(name = "cache_start")
private Long cacheStart;
@Column(name = "cache_end")
private Long cacheEnd;
@Column(name = "cache_refresh_time")
private Long cacheRefreshTime;
@Column(name = "cache_recalculate_time")
private Long cacheRecalculateTime;
@Column(name = "cache_refresh_status")
private Integer cacheRefreshStatus = CacheEnums.CacheRefreshStatus.stable.getCode();
@Column(name = "cache_recalculate_status")
private Integer cacheRecalculateStatus = CacheEnums.CacheRecalculateStatus.stable.getCode();
@Column(name = "keywords")
private String keywords;
@Column(name = "create_time_last")
private Long createTimeLast = System.currentTimeMillis();
@Column(name = "if_update")
private Integer ifUpdate = CacheEnums.UpdateSubject.no.getCode();
@Column(name = "update_num_left")
private Integer updateNumLeft = CacheEnums.UpdateSubjectNumLeft;
@Transient
private Integer companyRealQueryDays;
@Transient
private String token;
/**
* 已选择的站点
*/
@Transient
// private List<TopicWebsite> chooseWebsiteList;
//
// public List<TopicWebsite> getChooseWebsiteList() {
// return chooseWebsiteList;
// }
// public void setChooseWebsiteList(List<TopicWebsite> chooseWebsiteList) {
// this.chooseWebsiteList = chooseWebsiteList;
// }
public Integer getCompanyRealQueryDays() {
return companyRealQueryDays;
}
public void setCompanyRealQueryDays(Integer companyRealQueryDays) {
this.companyRealQueryDays = companyRealQueryDays;
}
public String getToken() {
return creator;
}
public void setToken(String token) {
this.token = token;
}
public Long getCacheStart() {
return cacheStart;
}
public void setCacheStart(Long cacheStart) {
this.cacheStart = cacheStart;
}
public Long getCacheEnd() {
return cacheEnd;
}
public void setCacheEnd(Long cacheEnd) {
this.cacheEnd = cacheEnd;
}
public Long getCacheRefreshTime() {
return cacheRefreshTime;
}
public void setCacheRefreshTime(Long cacheRefreshTime) {
this.cacheRefreshTime = cacheRefreshTime;
}
public Long getCacheRecalculateTime() {
return cacheRecalculateTime;
}
public void setCacheRecalculateTime(Long cacheRecalculateTime) {
this.cacheRecalculateTime = cacheRecalculateTime;
}
public Integer getCacheRefreshStatus() {
return cacheRefreshStatus;
}
public void setCacheRefreshStatus(Integer cacheRefreshStatus) {
this.cacheRefreshStatus = cacheRefreshStatus;
}
public Integer getCacheRecalculateStatus() {
return cacheRecalculateStatus;
}
public void setCacheRecalculateStatus(Integer cacheRecalculateStatus) {
this.cacheRecalculateStatus = cacheRecalculateStatus;
}
public String getKeywords() {
return keywords;
}
public void setKeywords(String keywords) {
this.keywords = keywords;
}
public Long getCreateTimeLast() {
return createTimeLast;
}
public void setCreateTimeLast(Long createTimeLast) {
this.createTimeLast = createTimeLast;
}
public Integer getIfUpdate() {
return ifUpdate;
}
public void setIfUpdate(Integer ifUpdate) {
this.ifUpdate = ifUpdate;
}
public Integer getUpdateNumLeft() {
return updateNumLeft;
}
public void setUpdateNumLeft(Integer updateNumLeft) {
this.updateNumLeft = updateNumLeft;
}
@Override
public String toString() {
return new ToStringBuilder(this)
.append("id", id)
.append("name", name)
.append("nodeType", nodeType)
.append("keywordsType", keywordsType)
.append("simpleContent", simpleContent)
.append("relationContent", relationContent)
.append("expertContent", expertContent)
.append("excludeContent", excludeContent)
.append("channels", channels)
.append("createTime", createTime)
.append("updateTime", updateTime)
.append("esQuery", esQuery)
.append("creator", creator)
.append("cacheStart", cacheStart)
.append("cacheEnd", cacheEnd)
.append("cacheRefreshTime", cacheRefreshTime)
.append("cacheRecalculateTime", cacheRecalculateTime)
.append("cacheRefreshStatus", cacheRefreshStatus)
.append("cacheRecalculateStatus", cacheRecalculateStatus)
.append("keywords", keywords)
.append("createTimeLast", createTimeLast)
.append("ifUpdate", ifUpdate)
.append("updateNumLeft", updateNumLeft)
.append("kwExpression", kwExpression)
.append("userIdList", userIdList)
.toString();
}
}

29
cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/BookMarkRepository.java
View File

@ -1,29 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.repository.mysql;
import com.bfd.mf.common.web.entity.mysql.BookMark;
import org.springframework.data.repository.CrudRepository;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public interface BookMarkRepository extends CrudRepository<BookMark, Long> {
List<BookMark> findAll();
}

13
cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/SentimentRepository.java
View File

@ -1,13 +0,0 @@
package com.bfd.mf.common.web.repository.mysql;
import com.bfd.mf.common.web.entity.mysql.SentimentModify;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.PagingAndSortingRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface SentimentRepository extends PagingAndSortingRepository<SentimentModify, Long>{
@Query(value = "select * from cl_sentiment where doc_id =?1",nativeQuery = true)
SentimentModify findByUniqueId(String docId);
}

469
cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/UserRepository.java
View File

@ -1,469 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.repository.mysql;
import com.bfd.mf.common.web.entity.mysql.User;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.PagingAndSortingRepository;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Repository;
@Repository
public interface UserRepository extends PagingAndSortingRepository<User, Long> {
// User findByUserNameAndDel(String userName, Boolean del);
//
// List<User> findByAccountAndDel(String account, Boolean del);
//
//
// @Query(value = "SELECT * FROM cl_user WHERE del = FALSE AND status = 1 AND account = ?1 ", nativeQuery = true)
// User findAvailableByAccount(String account);
//
// @Query(value = "SELECT * FROM cl_user WHERE del = FALSE AND account = ?1 ", nativeQuery = true)
// User findUserByAccount(String account);
//
// User findByUserNameAndPasswordAndDel(String userName, String pwd, Boolean del);
//
// User findByAccountAndPasswordAndDel(String account, String pwd, Boolean del);
//
// User findByCompanyIdAndUserNameAndDel(Long companyId, String userName, Boolean del);
//
// User findByCompanyIdAndAccountAndDel(Long companyId, String account, Boolean del);
User findByTokenAndDel(String token, Boolean del);
// @Query(value = "SELECT * FROM cl_user WHERE token = ?1 AND status = ?2", nativeQuery = true)
// User findByTokenAndStatus(String token, int status);
//
// User findByTokenAndStatusAndDel(String token, int status, Boolean del);
@Query(value = "SELECT company_id FROM cl_user WHERE token = :token AND del = :del", nativeQuery = true)
Long getCompanyIdByTokenAndDel(@Param("token") String token, @Param("del") Boolean del);
// @Query("select u from User u where u.roleId =?1 or u.roleId like '?1,%' or u.roleId like '%,?1' or u.roleId like " +
// "'%,?1,%' and del = false")
// List<User> findByRoleId(String rid);
//
// List<User> findByStatusAndCompanyIdAndDel(Integer status, Long companyId, Boolean del);
//
// List<User> findByCompanyIdAndDel(Long companyId, Boolean del);
//
//
// /**
// * Get all users with role and creator information
// *
// * @param companyId
// * @return
// */
// @Query(value = "SELECT " +
// "a.id , " +
// "a.token , " +
// "a.account , " +
// "a.userName , " +
// "r.id AS roleId , " +
// "r.name as roleName, " +
// "b.userName AS creator, " +
// "a.status, " +
// "a.email, " +
// "a.wechat, " +
// "a.wechatId, " +
// "a.mobile, " +
// "a.wechatVerified, " +
// "a.wechatOpenId " +
// "FROM " +
// "User a , " +
// "User b , " +
// "Role r " +
// "WHERE a.companyId = b.companyId " +
// "AND a.companyId = :cid " +
// "AND a.parentId = b.id " +
// "AND a.roleId = r.id " +
// "AND a.userName like :name " +
// "AND (a.del IS NULL OR a.del = false)" +
// "AND a.wechat IS NOT NULL " +
// "AND a.wechatId is NOT NULL " +
// "AND a.wechat <> '' " +
// "AND a.wechatId <> '' " +
// "AND a.wechatVerified in (:scope) " +
// "order by a.createdTime desc")
// Page<Object> findClientUserWeixinList(@Param("cid") Long companyId, @Param("name") String filterName, Pageable
// pageable, @Param("scope") List<String> scope);
//
//
// /**
// * Get all users with role and creator information
// *
// * @param companyId
// * @return
// */
// @Query(value = "SELECT " +
// "a.id , " +
// "a.token , " +
// "a.account , " +
// "a.userName , " +
// "r.id AS roleId , " +
// "r.name as roleName, " +
// "b.userName AS creator, " +
// "a.status, " +
// "a.email, " +
// "a.wechat, " +
// "a.wechatId, " +
// "a.mobile " +
// "FROM " +
// "User a , " +
// "User b , " +
// "Role r " +
// "WHERE a.companyId = b.companyId " +
// "AND a.companyId = :cid " +
// "AND a.parentId = b.id " +
// "AND a.roleId = r.id " +
// "AND a.userName like :name " +
// "AND (a.del IS NULL OR a.del = false)" +
// "AND a.email IS NOT NULL " +
// "AND a.email <> '' " +
// "order by a.createdTime desc")
// Page<Object> findClientUserEmailList(@Param("cid") Long companyId, @Param("name") String filterName, Pageable
// pageable);
//
// @Modifying
// @Query("update User set status = ?2, updatedTime = ?3 where id = ?1")
// int switchUserStatus(Long userId, Integer status, Long updateTime);
//
// @Modifying
// @Query("update User set del = ?2, updatedTime = ?3 where id = ?1")
// int deleteUser(Long userId, Boolean isDel, Long updateTime);
//
//
//// @Query("SELECT " +
//// "a.id ," +
//// "a.account , " +
//// "a.userName, " +
//// "a.email, " +
//// "a.wechat, " +
//// "a.wechatId, " +
//// "a.mobile, " +
//// "r.id AS userGroupId , " +
//// "r.name as userGroupName, " +
//// "a.status, " +
//// "a.createdTime " +
//// "FROM " +
//// "User a , " +
//// "UserGroup r " +
//// "WHERE a.id = :uid " +
//// "AND a.userGroupId = r.id " +
//// "AND (a.del = :isDel ) " +
//// "order by a.createdTime desc")
//// List<Object[]> fetchUser(@Param("uid") Long userId, @Param("isDel") Boolean isDel);
//
//
// /**
// * Check if reach user create limitation, return Y if avaliable
// * <p>
// * Return NULL if not available
// *
// * @param token
// * @return
// */
// @Query(value = "SELECT \"Y\" AS VALID " +
// "FROM " +
// " (SELECT company_user_count AS u_limit " +
// " FROM cl_company " +
// " WHERE status = 1 " +
// " AND id = " +
// " (SELECT company_id " +
// " FROM cl_user " +
// " WHERE token = ?1)) a, " +
// " (SELECT count(1) AS u_count " +
// " FROM cl_user " +
// " WHERE company_id = " +
// " (SELECT company_id " +
// " FROM cl_user " +
// " WHERE token = ?1) " +
// " AND del = FALSE) b " +
// "WHERE a.u_limit >= b.u_count", nativeQuery = true)
// List checkIfUserLimitReached(String token);
//
// /**
// * Check if user over dead date and open status, return true if available
// * <p>
// * Return false if not available
// *
// * @param token
// * @return
// */
// @Query(value =
// " SELECT \"Y\" AS flag " +
// " FROM cl_company " +
// " WHERE status = 1 " +
// " AND deadline_time < UNIX_TIMESTAMP() " +
// " AND id = " +
// " (SELECT company_id " +
// " FROM cl_user " +
// " WHERE token = ?1 and del = false and status = 1)", nativeQuery = true)
// List checkIfUserOverDeadDate(String token);
//
// /**
// * 返回话题上限数
// * <p>
// * SELECT company_topic_count FROM cl_company
// * WHERE id=(SELECT company_id FROM cl_user WHERE token = '8FB24E550815BCBA')
// *
// * @param token
// * @return
// */
// @Query(value =
// " SELECT company_topic_count " +
// " FROM cl_company " +
// " WHERE id = " +
// " (SELECT company_id" +
// " FROM cl_user " +
// " WHERE token = ?1)", nativeQuery = true)
// int getCompanyTopicCount(String token);
//
// /**
// * 返回创建话题的ids
// *
// * @param token
// * @return
// */
// @Query(value =
// " SELECT topic_ids" +
// " FROM cl_role " +
// " WHERE company_id = " +
// " (SELECT company_id" +
// " FROM cl_user " +
// " WHERE token = ?1) ", nativeQuery = true)
// List<String> getSujectIds(String token);
//
// /**
// * 返回客户创建的话题总数的结果
// * <p>
// * SELECT COUNT(DISTINCT(label_id)) FROM mf_subject
// * WHERE label_id IN(
// * SELECT topic_ids FROM mf_role
// * WHERE company_id ='14')
// *
// * @param labelIds
// * @return
// */
// @Query(value =
// " SELECT count(1) " +
// " FROM cl_subject " +
// " WHERE label_id IN " +
// " (?1)", nativeQuery = true)
// int getCompanyResult(Set<String> labelIds);
//
// @Query("select u from User u, Role r " +
// "where u.roleId = r.id and u.companyId =:cid and " +
// "r.isSystem >= (select r2.isSystem from Role r2 where r2.id = :rid) and u.del = false")
// List<User> getUserListByCompanyAndRole(@Param("cid") Long companyId, @Param("rid") Long roleId);
//
// @Query("select " +
// "u.account," +
// "wu.openId," +
// "wu.subscribe," +
// "wu.nickName," +
// "wu.sex," +
// "wu.language," +
// "wu.city," +
// "wu.province," +
// "wu.headimgUrl," +
// "wu.subscribeTime," +
// "wu.unionId," +
// "wu.remark," +
// "wu.grouId," +
// "wu.tagIdList," +
// "wu.mfRemark " +
// "from User u, Role r, WechatBind wb, WechatUser wu " +
// "where u.roleId = r.id and u.companyId =:cid and u.id = wb.userId and wb.openId = wu.openId and " +
// "r.isSystem >= (select r2.isSystem from Role r2 where r2.id = :rid) and u.del = false")
// List getUserInfoListByCompanyAndRole(@Param("cid") Long companyId, @Param("rid") Long roleId);
//
// @Query(value = "select * from cl_user where id =?1", nativeQuery = true)
// User getUserById(long creator);
//
// /**
// * 返回客户ID
// *
// * @param token
// * @return
// */
// @Query(value = "select company_id from cl_user where token =?1", nativeQuery = true)
// Integer findCompanyIdByToken(String token);
//
// /**
// * 返回有效客户
// *
// * @param id
// * @param flag
// * @return
// */
// @Query(value = "SELECT token FROM cl_user WHERE company_id=?1 AND del=?2", nativeQuery = true)
// List<String> findTokenByCompanyId(Long id, boolean flag);
//
// @Query(value = "SELECT token FROM cl_user WHERE company_id =?1 AND del=?2 AND STATUS=?3 limit 1", nativeQuery = true)
// String findEffectTokenByCompanyIdAndDel(Long companyId, boolean del, int status);
//
// @Query(value =
// "select company_id " +
// "from cl_user " +
// "where token = ?1 and del=?2", nativeQuery = true)
// Long findCompanyIdByTokenAndDel(String token, boolean del);
//
// @Query(value = "SELECT account FROM cl_user WHERE email=?1 AND status=?2", nativeQuery = true)
// String fetchUserOfAccountByReceiverAndStatus(String receiver, int status);
//
// @Query(value =
// " SELECT company_keyword_count " +
// " FROM cl_company " +
// " WHERE id = " +
// " (SELECT company_id" +
// " FROM cl_user " +
// " WHERE token = ?1)", nativeQuery = true)
// int getCompanyKeywordCount(String token);
//
// @Query(value = "SELECT company_exclude_count FROM cl_company WHERE id = (SELECT company_id FROM cl_user WHERE token = ?1)", nativeQuery = true)
// int getCompanyExcludeCount(String token);
//
// @Query(value = "SELECT role_id FROM cl_user WHERE token = :token AND del = :del", nativeQuery = true)
// String getRoleIdByTokenAndDel(@Param("token") String token, @Param("del") Boolean del);
//
// /**
// * Get all users with role and creator information
// *
// * @param companyId
// * @return
// */
// @Query(value = "SELECT " +
// "a.id , " +
// "a.account , " +
// "a.userName , " +
// "r.id AS userGroupId , " +
// "r.name as userGroupName, " +
// //"b.userName AS creator, " +
// " (case when a.parentId = 0 then '系统' when m.roleType = 1 then '管理员' else b.account end ) AS creator , " +
// "a.status, " +
// "a.email, " +
// "a.wechat, " +
// "a.wechatId, " +
// "a.mobile, " +
// "a.createdTime, "+
// "r.roleType "+
// "FROM " +
// "User a , " +
// "User b , " +
// "UserGroup m , " +
// "UserGroup r " +
// "WHERE " +
// //"a.companyId = b.companyId " +
// " a.companyId = :cid " +
// "AND a.userGroupId = r.id " +
// "AND a.userGroupId = :ugid " +
// "AND b.userGroupId = m.id " +
// "AND a.parentId = b.id " +
// "AND a.isShadow <> :isShadow " +
// //"AND a.roleId = r.id " +
// "AND (a.userName like :name or a.account like :name) " +
// "AND (a.del = :isDel) " +
// "order by a.createdTime desc" )
// Page<Object> findClientUserListByGroupId(@Param("cid") Long companyId, @Param("name") String filterName, Pageable pageable, @Param("ugid") Long userGroupId, @Param("isDel") Boolean isDel, @Param("isShadow") Integer isShadow);
//
// @Query(value = "SELECT * FROM cl_user WHERE account = ?1", nativeQuery = true)
// User findByAccount(String account);
//
// @Query(value = "SELECT * FROM cl_user WHERE user_group_id = ?1 and del = ?2", nativeQuery = true)
// List<User> findByUserGroupId(String userGroupId, Boolean del);
//
// @Query(value = "SELECT token FROM cl_user WHERE id IN (?1)", nativeQuery = true)
// List<String> getTokenListByIdList(List<Long> idList);
//
// @Query(value = "SELECT id FROM cl_user WHERE token IN (?1)", nativeQuery = true)
// List<Integer> getIdListByTokenList(List<String> tokenList);
//
// @Query(value = "SELECT " +
// "a.id , " +
// "a.account , " +
// "a.userName , " +
// "r.id AS userGroupId , " +
// "r.name as userGroupName, " +
// // "b.userName AS creator, " +
// " (case when a.parentId = 0 then '系统' when m.roleType = 1 then '管理员' else b.account end ) AS creator , " +
// "a.status, " +
// "a.email, " +
// "a.wechat, " +
// "a.wechatId, " +
// "a.mobile, " +
// "a.createdTime, " +
// "r.roleType " +
// "FROM " +
// "User a , " +
// "User b , " +
// "UserGroup m , " +
// "UserGroup r " +
// "WHERE " +
// //"a.companyId = b.companyId " +
// "a.companyId = :cid " +
// "AND a.parentId = b.id " +
// "AND a.userGroupId = r.id " +
// "AND b.userGroupId = m.id " +
// "AND a.isShadow <> :isShadow " +
// "AND r.roleType not in :ugrt " +
// "AND (a.userName like :name or a.account like :name) " +
// "AND (a.del = :isDel ) " +
// "order by a.createdTime desc")
// Page<Object> findClientUserListFilterByRole(@Param("cid") Long companyId,
// @Param("name") String filterName,
// Pageable pageable,
// @Param("isDel") Boolean isDel,
// @Param("ugrt") List<Integer> userGroupRoleType,
// @Param("isShadow") Integer isShadow);
// @Query(value = "SELECT id,account,user_group_id FROM cl_user WHERE company_id = ?1 and del = ?2 ORDER BY created_time DESC ", nativeQuery = true)
// List<Object[]> findAllByCompanyId(Long companyId, boolean del);
//
// @Query(value = "SELECT * FROM cl_user WHERE id = ?1 AND status = ?2 AND del = ?3", nativeQuery = true)
// User findByIdAndStatusAndDel(Long userId, Integer userNormal, Boolean userNotDelete);
//
// @Query(value = "SELECT * FROM cl_user WHERE token = ?1 ", nativeQuery = true)
// User findBytoken(String creator);
// @Query("SELECT " +
// "a.id ," +
// "a.account , " +
// "a.userName, " +
// "a.email, " +
// "a.wechat, " +
// "a.wechatId, " +
// "a.mobile, " +
// "r.id AS userGroupId , " +
// "r.name as userGroupName, " +
// "a.status, " +
// "a.createdTime, " +
// "a.password " +
// "FROM " +
// "User a , " +
// "UserGroup r " +
// "WHERE a.token = :token " +
// "AND a.userGroupId = r.id " +
// "AND (a.del = :isDel ) " +
// "order by a.createdTime desc")
// List<Object[]> fetchUserByTokenAndDel(@Param("token") String token, @Param("isDel") Boolean userNotDelete);
// @Query(value = "select g.channel_ids from cl_user u, cl_user_group g where u.token = ?1 AND g.id = u.user_group_id ", nativeQuery = true)
// String getChannelIdsByToken(String token);
}

13
cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/base/GenericDao.java
View File

@ -1,13 +0,0 @@
package com.bfd.mf.common.web.repository.mysql.base;
import com.bfd.mf.common.web.entity.mysql.base.BaseEntityModel;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.data.repository.NoRepositoryBean;
import java.io.Serializable;
@NoRepositoryBean
public interface GenericDao<T extends BaseEntityModel, PK extends Serializable>
extends JpaRepository<T, PK>, JpaSpecificationExecutor<T> {
}

13
cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/base/GenericTreeDao.java
View File

@ -1,13 +0,0 @@
package com.bfd.mf.common.web.repository.mysql.base;
import com.bfd.mf.common.web.entity.mysql.base.BaseTreeEntityModel;
import org.springframework.data.repository.NoRepositoryBean;
import java.io.Serializable;
import java.util.List;
@NoRepositoryBean
public interface GenericTreeDao<T extends BaseTreeEntityModel<T>, PK extends Serializable>
extends GenericDao<T, PK> {
List<T> getRoot();
}

25
cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/cache/ClusterRepository.java
View File

@ -1,25 +0,0 @@
package com.bfd.mf.common.web.repository.mysql.cache;
import com.bfd.mf.common.web.entity.mysql.cache.Cluster;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.CrudRepository;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public interface ClusterRepository extends CrudRepository<Cluster, Long> {
Cluster findClusterByType(int type);
/**
* 返回集群id
*/
@Query(value = "SELECT * FROM cl_cluster WHERE type IN(?1) ", nativeQuery = true)
List<Cluster> findClusterByTypeList(List<Integer> typeList);
/**
* 集群
*/
@Query(value = "SELECT * FROM cl_cluster WHERE type=?1 ", nativeQuery = true)
List<Cluster> findClusterListByType(int subjectClusterType);
}

12
cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/cache/SubjectClusterRepository.java
View File

@ -1,12 +0,0 @@
package com.bfd.mf.common.web.repository.mysql.cache;
import com.bfd.mf.common.web.entity.mysql.cache.SubjectCluster;
import org.springframework.data.repository.CrudRepository;
import org.springframework.stereotype.Repository;
import java.math.BigInteger;
@Repository
public interface SubjectClusterRepository extends CrudRepository<SubjectCluster, Long> {
SubjectCluster findSubjectClusterBySubjectId(BigInteger subjectId);
}

14
cl_search_api/src/main/java/com/bfd/mf/common/web/repository/mysql/report/MainSourceRepository.java
View File

@ -1,14 +0,0 @@
package com.bfd.mf.common.web.repository.mysql.report;
import com.bfd.mf.common.web.entity.mysql.report.MainSourceEntity;
import org.springframework.data.repository.CrudRepository;
import org.springframework.stereotype.Repository;
import java.util.List;
@Repository
public interface MainSourceRepository extends CrudRepository<MainSourceEntity, Long> {
List<MainSourceEntity> findByCompanyIdAndStatus(long companyId, int status);
}

59
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/BaseRequest.java
View File

@ -1,59 +0,0 @@
package com.bfd.mf.common.web.vo.params;
import lombok.NoArgsConstructor;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.hibernate.validator.constraints.NotEmpty;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotNull;
import java.io.Serializable;
@NoArgsConstructor
public class BaseRequest implements Serializable {
private static final long serialVersionUID = -8591843291887513043L;
@NotEmpty(message = "Param 'token' can't be NULL")
@NotNull
private String token;
@NotEmpty(message = "Param 'sign' can't be NULL")
private String sign;
@Min(value = 1, message = "Param 'time' is invalid")
private Long time;
@Override
public String toString() {
return new ToStringBuilder(this)
.append("token", token)
.append("sign", sign)
.append("time", time)
.toString();
}
public String getToken() {
return token;
}
public void setToken(String token) {
this.token = token;
}
public String getSign() {
return sign;
}
public void setSign(String sign) {
this.sign = sign;
}
public Long getTime() {
return time;
}
public void setTime(Long time) {
this.time = time;
}
}

259
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/QueryRequest.java
View File

@ -1,259 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.vo.params;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
public class QueryRequest implements Serializable {
private static final long serialVersionUID = 1L;
// 必传字段
private String subjectId;
private Long userId;
// 翻页字段
private Integer page;
private Integer limit;
//其他参数
private Long pubTime;
private String docId;
private String docIds;
private String siteTypes; //站点类型 必传，多个,分隔， 全部传“”
private String order; // 排序方式 asc/desc
private String sidx; // 排序字段 pubTime crawlTime ...
private String searchArea; //0 美国1中国……
private String cid; // 站点名
private String crawlDataFlag ; // 数据标识
private Integer searchScope; // 0 content 1 title 2 author 3 con+tit 4con+aut 5con+com
private String keyword; // 要查询的关键词
private Long crawlStartTime;
private Long crawlEndTime;
private Long startTime;
private Long endTime;
public static long getSerialVersionUID() {
return serialVersionUID;
}
public String getSubjectId() {
return subjectId;
}
public void setSubjectId(String subjectId) {
this.subjectId = subjectId;
}
public Long getUserId() {
return userId;
}
public void setUserId(Long userId) {
this.userId = userId;
}
public Integer getPage() {
return page;
}
public void setPage(Integer page) {
this.page = page;
}
public Integer getLimit() {
return limit;
}
public void setLimit(Integer limit) {
this.limit = limit;
}
public Long getPubTime() {
return pubTime;
}
public void setPubTime(Long pubTime) {
this.pubTime = pubTime;
}
public String getDocId() {
return docId;
}
public void setDocId(String docId) {
this.docId = docId;
}
public String getDocIds() {
return docIds;
}
public void setDocIds(String docIds) {
this.docIds = docIds;
}
public String getSiteTypes() {
return siteTypes;
}
public void setSiteTypes(String siteTypes) {
this.siteTypes = siteTypes;
}
public String getOrder() {
return order;
}
public void setOrder(String order) {
this.order = order;
}
public String getSidx() {
return sidx;
}
public void setSidx(String sidx) {
this.sidx = sidx;
}
public String getSearchArea() {
return searchArea;
}
public void setSearchArea(String searchArea) {
this.searchArea = searchArea;
}
public String getCid() {
return cid;
}
public void setCid(String cid) {
this.cid = cid;
}
public String getCrawlDataFlag() {
return crawlDataFlag;
}
public void setCrawlDataFlag(String crawlDataFlag) {
this.crawlDataFlag = crawlDataFlag;
}
public Integer getSearchScope() {
return searchScope;
}
public List<String> getSearchScopeValue(Integer num){
String result = Integer.toBinaryString(num);
// 10000 正文标题 正文内容 正文作者 评论标题 评论作者
List<Integer> valueList = new ArrayList<>();
for (int i = 0 ; i < result.length() ; i ++){
Integer oneNum = Integer.parseInt(result.substring(i,i+1));
valueList.add(oneNum);
}
List<String> resultList = new ArrayList<>();
if(valueList.size() >0 && valueList.get(0) == 1){
resultList.add("title");
}
if(valueList.size() >1 && valueList.get(1) ==1){
resultList.add("content");
}
if(valueList.size() >2 && valueList.get(2) ==1){
resultList.add("author");
}
if(valueList.size() >3 && valueList.get(3) ==1){
resultList.add("title");
}
if(valueList.size() >4 && valueList.get(4) ==1){
resultList.add("author");
}
return resultList;
}
public void setSearchScope(Integer searchScope) {
this.searchScope = searchScope;
}
public String getKeyword() {
return keyword;
}
public void setKeyword(String keyword) {
this.keyword = keyword;
}
public Long getCrawlStartTime() {
return crawlStartTime;
}
public void setCrawlStartTime(Long crawlStartTime) {
this.crawlStartTime = crawlStartTime;
}
public Long getCrawlEndTime() {
return crawlEndTime;
}
public void setCrawlEndTime(Long crawlEndTime) {
this.crawlEndTime = crawlEndTime;
}
public Long getStartTime() {
return startTime;
}
public void setStartTime(Long startTime) {
this.startTime = startTime;
}
public Long getEndTime() {
return endTime;
}
public void setEndTime(Long endTime) {
this.endTime = endTime;
}
@Override
public String toString() {
return "QueryRequest{" +
"subjectId='" + subjectId + '\'' +
", userId=" + userId +
", page=" + page +
", limit=" + limit +
", pubTime=" + pubTime +
", docId='" + docId + '\'' +
", docIds='" + docIds + '\'' +
", siteTypes='" + siteTypes + '\'' +
", order='" + order + '\'' +
", sidx='" + sidx + '\'' +
", searchArea='" + searchArea + '\'' +
", cid='" + cid + '\'' +
", crawlDataFlag='" + crawlDataFlag + '\'' +
", searchScope=" + searchScope +
", keyword='" + keyword + '\'' +
", crawlStartTime=" + crawlStartTime +
", crawlEndTime=" + crawlEndTime +
", startTime=" + startTime +
", endTime=" + endTime +
'}';
}
}

51
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/expression/DistanceNode.java
View File

@ -1,51 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.vo.params.expression;
/**
* @author lei.bao Date: 2016年10月13日
*/
public class DistanceNode {
public final char opera = '~';// ~
private int distance;// 例如10
private boolean formatFinish = false;
public DistanceNode(int dis) {
this.distance = dis;
}
public int getDistance() {
return distance;
}
public void setDistance(int distance) {
this.distance = distance;
}
public boolean isFormatFinish() {
return formatFinish;
}
public void setFormatFinish(boolean formatFinish) {
this.formatFinish = formatFinish;
}
@Override
public String toString() {
return "opera='~',distance=" + distance;
}
}

114
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/expression/ExpNode.java
View File

@ -1,114 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.vo.params.expression;
public class ExpNode {
private boolean isOpera;// 操作符还是操作数
private Object value;// ExpObject 或者OperaObject
private ExpNode parent;
private ExpNode left;
private ExpNode right;
private boolean isLeaf;
public ExpNode(Object c) {
this.value = c;
// this.parent = null;
// this.left = null;
// this.right = null;
// this.isLeaf = true;
}
public ExpNode(Object c, ExpNode p) {
this(c);
this.parent = p;
}
public boolean isOpera() {
return isOpera;
}
public void setOpera(boolean isOpera) {
this.isOpera = isOpera;
}
public Object getValue() {
return value;
}
public void setValue(Object value) {
this.value = value;
}
public ExpNode getParent() {
return parent;
}
public void setParent(ExpNode parent) {
this.parent = parent;
}
public ExpNode getLeft() {
return left;
}
public void setLeft(ExpNode left) {
this.left = left;
}
public ExpNode getRight() {
return right;
}
public void setRight(ExpNode right) {
this.right = right;
}
public boolean isLeaf() {
return isLeaf;
}
public void setLeaf(boolean isLeaf) {
this.isLeaf = isLeaf;
}
/**
*
* @param root
* @return
*/
public static ExpNode clone(ExpNode root) {
if (null == root || null == root.getValue()) {
return null;
}
ExpNode node = new ExpNode(root.getValue());
node.setLeaf(root.isLeaf);
node.setOpera(root.isOpera);
// ExpNode parent = clone(root.getParent());
// node.setParent(parent);
if (null == root.getLeft() && null == root.getRight()) {
return node;
}
ExpNode left = clone(root.getLeft());
ExpNode right = clone(root.getRight());
left.setParent(node);
right.setParent(node);
//
node.setLeft(left);
node.setRight(right);
return node;
}
}

108
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/expression/ExpObject.java
View File

@ -1,108 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.vo.params.expression;
import java.util.ArrayList;
import java.util.List;
/**
* 最小操作对象单元(不包含嵌套表达式)
*
* @author lei.bao<br>
* @Date: 2016年10月12日<br>
*/
public class ExpObject {
public static final String[] defaultFields = new String[] { "title", "content" };
private String[] fields;
private String expression;
private int slop = 0;// 默认slop
private List<String> tokens = new ArrayList<>();// 适用于多个字符组成的词组,如"apple pay"
public ExpObject(String v) {
this.expression = v.trim();
this.fields = defaultFields;
}
public ExpObject(String v, String[] fields) {
this.expression = v.trim();
this.fields = fields;
}
public ExpObject(String v, int slop,List<String> tk) {
this(v);
this.slop = slop;
this.tokens = tk;
}
public String getExpression() {
return expression;
}
public void setExpression(String expression) {
this.expression = expression;
}
public static String[] getDefaultfields() {
return defaultFields;
}
public String[] getFields() {
return fields;
}
public void setFields(String[] fields) {
this.fields = fields;
}
public int getSlop() {
return slop;
}
public void setSlop(int slop) {
this.slop = slop;
}
public List<String> getTokens() {
return tokens;
}
public void setTokens(List<String> tokens) {
this.tokens = tokens;
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("{").append("value:").append(expression).append(",").append("fields:").append(getFieldStr(fields))
.append("}");
return sb.toString();
}
private String getFieldStr(String[] field) {
if (null == field || field.length == 0) {
return "[null]";
}
StringBuilder sb = new StringBuilder();
sb.append("[");
for (int i = 0; i < field.length; i++) {
sb.append(field[i]);
if (i < field.length - 1) {
sb.append(",");
}
}
sb.append("]");
return sb.toString();
}
}

55
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/expression/HalfQuartChar.java
View File

@ -1,55 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.vo.params.expression;
/**
* @author lei.bao<br>
* @Date: 2016年10月13日
*/
public class HalfQuartChar {
private char charC;//半边符号,如"<"，"(",等待另外半边匹配
private int position;
public HalfQuartChar() {
}
public HalfQuartChar(char lf, int pos) {
this.charC = lf;
this.position = pos;
}
public char getCharC() {
return charC;
}
public void setCharC(char charC) {
this.charC = charC;
}
public int getPosition() {
return position;
}
public void setPosition(int position) {
this.position = position;
}
@Override
public String toString() {
return new String(new char[]{charC});
}
}

78
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/expression/OperaObject.java
View File

@ -1,78 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.vo.params.expression;
/**
* @author lei.bao Date: 2016年10月13日
*/
public class OperaObject {
public static final int OPERA_TYPE_ONE = 1;
public static final int OPERA_TYPE_TWO = 2;
private int type = OPERA_TYPE_ONE;// 默认一元表达式
private char opera;
private Object obj;// 一元表达式操作对象,如~10中的10，！A中的A
public OperaObject(char ch) {
this.opera = ch;
if (ch == '!' || ch == '！' || ch == '-') {
this.type = OPERA_TYPE_ONE;
} else {
this.type = OPERA_TYPE_TWO;
}
}
public OperaObject(char ch, int type) {
this.opera = ch;
this.type = type;
}
public OperaObject(char ch, Object obj) {
this.opera = ch;
this.type = OPERA_TYPE_ONE;
this.obj = obj;
}
public int getType() {
return type;
}
public void setType(int type) {
this.type = type;
}
public char getOpera() {
return opera;
}
public void setOpera(char opera) {
this.opera = opera;
}
public Object getObj() {
return obj;
}
public void setObj(Object obj) {
this.obj = obj;
}
@Override
public String toString() {
return "opera=" + opera + ",type=" + type;
}
}

249
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/post/PostMonitorRequest.java
View File

@ -1,249 +0,0 @@
package com.bfd.mf.common.web.vo.params.post;
import com.bfd.mf.common.web.vo.params.BaseRequest;
import com.bfd.nlp.common.util.object.TObjectUtils;
import com.bfd.nlp.common.util.string.TStringUtils;
import javax.validation.constraints.NotNull;
import java.io.Serializable;
import java.util.List;
/**
* 帖子分析参数类
*
* @author renpeng.wei
* @date 2018/8/1 15:42
*/
public class PostMonitorRequest extends BaseRequest implements Serializable {
@NotNull(message = "Param 'taskId' can't be NULL")
private String taskId;
private int level = 0;
private Long startTime;
private Long endTime;
/**
* common 正面 negative 负面 neuter 中性
***/
private List<String> emotion;
/**
* 未认证 0 个人认证 1 企业认证 2 未知 3 全部 4
***/
private List<String> verified;
private String keyWord;
/**
* quoteCount 贡献转发量 fansCount 作者粉丝量 commentsCount 评论量 attitudesCount 获赞量 pubTime 发布时间
***/
private String sort;
private int page = 0;
private int size = 10;
/**
* 1 转发列表 2 评论列表 4 转发分析结果 5转发词云列表 6评论分析结果 7 评论词云列表
**/
private int type;
/**
* 1 转发 2 评论
**/
private int style;
private Long nodeId;
private Integer allNumber;
/***
* 1 所有 2 地名 3 话题 4 表情 5 评论
*
*/
private Integer weiBoCloudType = 0;
private String weiBoCloudKeyword ;
public String getWeiBoCloudKeyword() {
return weiBoCloudKeyword;
}
public void setWeiBoCloudKeyword(String weiBoCloudKeyword) {
this.weiBoCloudKeyword = weiBoCloudKeyword;
}
public Integer getWeiBoCloudType() {
return weiBoCloudType;
}
public void setWeiBoCloudType(Integer weiBoCloudType) {
this.weiBoCloudType = weiBoCloudType;
}
public Long getNodeId() {
return nodeId;
}
public void setNodeId(Long nodeId) {
this.nodeId = nodeId;
}
public String getTaskId() {
return taskId;
}
public void setTaskId(String taskId) {
this.taskId = taskId;
}
public int getLevel() {
return level;
}
public void setLevel(int level) {
this.level = level;
}
public Long getStartTime() {
return startTime;
}
public void setStartTime(Long startTime) {
this.startTime = startTime;
}
public Long getEndTime() {
return endTime;
}
public void setEndTime(Long endTime) {
this.endTime = endTime;
}
public List<String> getEmotion() {
return emotion;
}
public void setEmotion(List<String> emotion) {
this.emotion = emotion;
}
public List<String> getVerified() {
return verified;
}
public void setVerified(List<String> verified) {
this.verified = verified;
}
public String getKeyWord() {
return keyWord;
}
public void setKeyWord(String keyWord) {
this.keyWord = keyWord;
}
public String getSort() {
return sort;
}
public void setSort(String sort) {
this.sort = sort;
}
public int getPage() {
return page;
}
public void setPage(int page) {
this.page = page;
}
public int getSize() {
return size;
}
public void setSize(int size) {
this.size = size;
}
public int getType() {
return type;
}
public void setType(int type) {
this.type = type;
}
public int getStyle() {
return style;
}
public void setStyle(int style) {
this.style = style;
}
public Integer getAllNumber() {
return allNumber;
}
public void setAllNumber(Integer allNumber) {
this.allNumber = allNumber;
}
@Override
public String toString() {
return "PostMonitorRequest{" +
"taskId='" + taskId + '\'' +
", level=" + level +
", startTime=" + startTime +
", endTime=" + endTime +
", emotion=" + emotion +
", verified=" + verified +
", keyWord='" + keyWord + '\'' +
", sort='" + sort + '\'' +
", page=" + page +
", size=" + size +
", type=" + type +
", style=" + style +
", nodeId=" + nodeId +
", allNumber=" + allNumber +
'}';
}
public String convertEmotiontoDbString() {
if (!TObjectUtils.isListEmpty(emotion)) {
return TStringUtils.join(emotion, ",");
}
return null;
}
public PostMonitorRequest() {
}
// public PostMonitorRequest(DataExportList dataExportList) {
// if (TObjectUtils.nonNull(dataExportList)) {
// this.setToken(dataExportList.getToken());
// this.setNodeId(dataExportList.getNodeId());
// if (TStringUtils.isNotEmpty(dataExportList.getEmotionList())) {
// this.setEmotion(Arrays.asList(dataExportList.getEmotionList().split(",")));
// }
// this.setEndTime(dataExportList.getTimeEnd());
// this.setStartTime(dataExportList.getTimeStart());
// this.setKeyWord(dataExportList.getSearchText());
// if(dataExportList.getType().equals(ExportEnum.QUOTE.getCode())) {
// this.setStyle(UserConstant.EXPORT_WEIBO_TYPE_QUOTE);
// }else{
// this.setStyle(UserConstant.EXPORT_WEIBO_TYPE_COMMENT);
// }
// this.setTaskId(dataExportList.getTaskId());
// this.setVerified(Arrays.asList(dataExportList.getVerified().split(",")));
// this.setAllNumber(dataExportList.getAllNumber());
// }
// }
}

54
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/spam/SpamConstant.java
View File

@ -1,54 +0,0 @@
package com.bfd.mf.common.web.vo.params.spam;
import java.util.HashSet;
import java.util.Set;
public class SpamConstant {
public static final String SPAM_TAG = "spamTag";
/**
* 出租
*/
private static final String RENT_TAG_EN = "rent";
/**
* 婚恋交友 marriageSeeking
*/
private static final String MARRIAGE_SEEKING_TAG_EN = "marriageSeeking";
/**
* 招聘
*/
private static final String RECRUITMENT_TAG_EN = "recruitment";
/**
* 表示正常的数据，其实不过滤
*/
public static final String NORMAL_TAG_EN = "other";
/**
* 黄赌毒
*/
private static final String SEXY_GAMBLE_DRUG_TAG_EN = "sexyGambleDrug";
/**
* 其他
*/
private static final String AD_TAG_EN = "ad";
/**
* 闲置转让
*/
private static final String IDLE_TRANSFER = "transfer";
public static final Set<String> INCLUDE_SPAM_TAG_SET_EN = new HashSet<>();
static {
INCLUDE_SPAM_TAG_SET_EN.add(RENT_TAG_EN);
INCLUDE_SPAM_TAG_SET_EN.add(MARRIAGE_SEEKING_TAG_EN);
INCLUDE_SPAM_TAG_SET_EN.add(RECRUITMENT_TAG_EN);
INCLUDE_SPAM_TAG_SET_EN.add(IDLE_TRANSFER);
}
}

16
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/params/womanalysis/ProductTag.java
View File

@ -1,16 +0,0 @@
package com.bfd.mf.common.web.vo.params.womanalysis;
import lombok.Data;
/**
* @Auther: huiyz
* @Date: 2018/11/23 10:09
* @Description:
*/
@Data
public class ProductTag {
private Long id;
private String name;
}

94
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/DataCompareDocCount.java
View File

@ -1,94 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.vo.view.analysis;
import com.alibaba.fastjson.JSONArray;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
/**
* 对比分析：声量、覆盖媒体声量、情感分析
*
* @author quanyou.chang
* @create 2016-09-21 12:55
*/
public class DataCompareDocCount extends DataCountBase implements Serializable {
/**
* 整体数量
*/
private List<DataPieCount> totalDocList = new ArrayList<>();
/**
* 覆盖媒体数量
*/
private List<DataPieCount> mediaDocList = new ArrayList<>();
/**
* 情感分析
*/
private List<NodeEmotionCount> nodeEmotionCountList = new ArrayList<>();
//对比分析--->趋势分析数据
private List<DataTrendCompare> subjectTrendList = new ArrayList<>();
private JSONArray timeAbscissa = new JSONArray();
public List<DataTrendCompare> getSubjectTrendList() {
return subjectTrendList;
}
public void setSubjectTrendList(List<DataTrendCompare> subjectTrendList) {
this.subjectTrendList = subjectTrendList;
}
public JSONArray getTimeAbscissa() {
return timeAbscissa;
}
public void setTimeAbscissa(JSONArray timeAbscissa) {
this.timeAbscissa = timeAbscissa;
}
public List<DataPieCount> getTotalDocList() {
return totalDocList;
}
public void setTotalDocList(List<DataPieCount> totalDocList) {
this.totalDocList = totalDocList;
}
public List<DataPieCount> getMediaDocList() {
return mediaDocList;
}
public void setMediaDocList(List<DataPieCount> mediaDocList) {
this.mediaDocList = mediaDocList;
}
public List<NodeEmotionCount> getNodeEmotionCountList() {
return nodeEmotionCountList;
}
public void setNodeEmotionCountList(List<NodeEmotionCount> nodeEmotionCountList) {
this.nodeEmotionCountList = nodeEmotionCountList;
}
}

37
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/DataCountBase.java
View File

@ -1,37 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.vo.view.analysis;
import org.apache.commons.lang3.builder.ToStringBuilder;
import org.apache.commons.lang3.builder.ToStringStyle;
import java.io.Serializable;
/**
* 数据统计基类
*
* @author quanyou.chang
* @create 2016-09-21 11:23
**/
public class DataCountBase implements Serializable {
@Override
public String toString() {
return ToStringBuilder.reflectionToString(this, ToStringStyle.MULTI_LINE_STYLE);
}
}

88
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/DataLineCount.java
View File

@ -1,88 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.vo.view.analysis;
/**
* 折线图统计
*
* @author quanyou.chang
* @create 2016-09-22 10:43
**/
public class DataLineCount extends DataCountBase implements Comparable<DataLineCount> {
/**
* 时间戳
*/
private Long timestamp;
/**
* 值
*/
private Long data;
public Long getTimestamp() {
return timestamp;
}
public void setTimestamp(Long timestamp) {
this.timestamp = timestamp;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
/**
* 坐标
*/
private String name;
public Long getData() {
return data;
}
public void setData(Long data) {
this.data = data;
}
@Override
public int compareTo(DataLineCount o) {
return this.getTimestamp().compareTo(o.getTimestamp());
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
DataLineCount that = (DataLineCount) o;
if (timestamp != null ? !timestamp.equals(that.timestamp) : that.timestamp != null) return false;
return name != null ? name.equals(that.name) : that.name == null;
}
@Override
public int hashCode() {
int result = timestamp != null ? timestamp.hashCode() : 0;
result = 31 * result + (name != null ? name.hashCode() : 0);
return result;
}
}

97
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/DataPieCount.java
View File

@ -1,97 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.vo.view.analysis;
/**
* 数据饼图占比
*
* @author quanyou.chang
* @create 2016-09-21 11:20
*/
public class DataPieCount extends DataCountBase implements Comparable<DataPieCount> {
public DataPieCount(String name, Long value) {
this.name = name;
this.value = value;
}
public DataPieCount(String name, Long value, String channel) {
this.name = name;
this.value = value;
this.channel=channel;
}
public DataPieCount(String name, Long value, Long nodeId) {
this.name = name;
this.value = value;
this.nodeId = nodeId;
}
private String name;
public DataPieCount() {
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Long getValue() {
return value;
}
public void setValue(Long value) {
this.value = value;
}
public String getChannel() {
return channel;
}
public void setChannel(String channel) {
this.channel = channel;
}
private Long value = 0L;
private Long nodeId;
private String channel;
private Double percent;
public Double getPercent() {
return percent;
}
public void setPercent(Double percent) {
this.percent = percent;
}
@Override
public int compareTo(DataPieCount o) {
return -(this.getValue().intValue() - o.getValue().intValue());
}
public Long getNodeId() {
return nodeId;
}
public void setNodeId(Long nodeId) {
this.nodeId = nodeId;
}
}

82
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/DataTrendCompare.java
View File

@ -1,82 +0,0 @@
package com.bfd.mf.common.web.vo.view.analysis;
import com.alibaba.fastjson.JSONArray;
import java.io.Serializable;
import java.util.List;
/**
* Created by BFD-194 on 2017/11/7.
*/
public class DataTrendCompare implements Serializable {
private String name;
private Long nodeId;
private JSONArray listNum;
public DataTrendCompare() {
}
public Long getNodeId() {
return nodeId;
}
public void setNodeId(Long nodeId) {
this.nodeId = nodeId;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public List getListNum() {
return listNum;
}
public void setListNum(JSONArray listNum) {
this.listNum = listNum;
}
public DataTrendCompare(String name, JSONArray listNum, int size) {
if (listNum.size() < size) {
JSONArray jsonArray = new JSONArray();
for (int i = 0; i < size; i++) {
jsonArray.add(i, 0);
}
this.listNum = jsonArray;
} else {
this.listNum = listNum;
}
this.name = name;
}
public DataTrendCompare(String name, JSONArray listNum, JSONArray abscissa, JSONArray timeCompare, Long nodeId) {
if (null == timeCompare || timeCompare.size() <= 0) {
this.listNum = listNum;
} else {
JSONArray jsonArray = new JSONArray();
for (int i = 0; i < timeCompare.size(); i++) {
boolean existTime = false;
for (int j = 0; j < abscissa.size(); j++) {
if (abscissa.get(j).equals(timeCompare.get(i))) {
jsonArray.add(i, listNum.get(j));
existTime = true;
break;
}
}
if (!existTime) jsonArray.add(i, 0);
}
this.listNum = jsonArray;
}
this.name = name;
this.nodeId = nodeId;
}
}

90
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/DataTrendCount.java
View File

@ -1,90 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.vo.view.analysis;
import java.util.List;
import java.util.Map;
/**
* 数据趋势声量统计
*
* @author quanyou.chang
* @create 2016-09-20 13:36
*/
public class DataTrendCount extends DataCountBase {
/**
* 整体声量
*/
private Long totalNumber = 0L;
/**
* 整体覆盖媒体
*/
private Long totalMediaNumber = 0L;
/**
* 敏感声量
*/
private Long negativeNumber = 0L;
public Long getTotalNumber() {
return totalNumber;
}
public void setTotalNumber(Long totalNumber) {
this.totalNumber = totalNumber;
}
public Long getTotalMediaNumber() {
return totalMediaNumber;
}
public void setTotalMediaNumber(Long totalMediaNumber) {
this.totalMediaNumber = totalMediaNumber;
}
public Long getNegativeNumber() {
return negativeNumber;
}
public void setNegativeNumber(Long negativeNumber) {
this.negativeNumber = negativeNumber;
}
public Long getNegativeMediaNumber() {
return negativeMediaNumber;
}
public void setNegativeMediaNumber(Long negativeMediaNumber) {
this.negativeMediaNumber = negativeMediaNumber;
}
/**
* 敏感覆盖媒体
*/
private Long negativeMediaNumber = 0L;
private Map<Long, List<Long>> mapNodeId;
public Map<Long, List<Long>> getMapNodeId() {
return mapNodeId;
}
public void setMapNodeId(Map<Long, List<Long>> mapNodeId) {
this.mapNodeId = mapNodeId;
}
}

73
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/NodeEmotionCount.java
View File

@ -1,73 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.vo.view.analysis;
import java.util.ArrayList;
import java.util.List;
/**
* @author quanyou.chang
* @create 2016-09-21 11:20
**/
public class NodeEmotionCount extends DataCountBase {
public NodeEmotionCount() {
}
public NodeEmotionCount(String nodeName, List<DataPieCount> dataPieCountList) {
this.nodeName = nodeName;
this.dataPieCountList = dataPieCountList;
}
public NodeEmotionCount(String nodeName, List<DataPieCount> dataPieCountList, Long nodeId) {
this.nodeName = nodeName;
this.dataPieCountList = dataPieCountList;
this.nodeId = nodeId;
}
private List<DataPieCount> dataPieCountList = new ArrayList<>();
private Long nodeId;
public String getNodeName() {
return nodeName;
}
public void setNodeName(String nodeName) {
this.nodeName = nodeName;
}
private String nodeName = null;
public List<DataPieCount> getDataPieCountList() {
return dataPieCountList;
}
public void setDataPieCountList(List<DataPieCount> dataPieCountList) {
this.dataPieCountList = dataPieCountList;
}
public Long getNodeId() {
return nodeId;
}
public void setNodeId(Long nodeId) {
this.nodeId = nodeId;
}
}

52
cl_search_api/src/main/java/com/bfd/mf/common/web/vo/view/analysis/PieCount.java
View File

@ -1,52 +0,0 @@
/*
* Copyright (C) 2016 Baifendian Corporation
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bfd.mf.common.web.vo.view.analysis;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
/**
* 饼图展示
*
* @author quanyou.chang
* @create 2016-09-21 12:55
**/
public class PieCount implements Serializable {
public List<DataPieCount> getChannelCounts() {
return channelCounts;
}
public void setChannelCounts(List<DataPieCount> channelCounts) {
this.channelCounts = channelCounts;
}
List<DataPieCount> channelCounts = new ArrayList<DataPieCount>();
public List<DataPieCount> getEmotionCounts() {
return emotionCounts;
}
public void setEmotionCounts(List<DataPieCount> emotionCounts) {
this.emotionCounts = emotionCounts;
}
List<DataPieCount> emotionCounts = new ArrayList<DataPieCount>();
}

Some files were not shown because too many files changed in this diff

Write Preview
Loading…
Cancel
Save