- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.bfd.mf.common.util;
-
-import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class ESConstant {
-
-// public static final String COMPANY_ID = "companyId";
-// public static final String INDUSTRY_ID = "industryId";
-
- public static final String SPAM_TAG = "spamTag";
-
- public static final String MEDIA_AREA_KEY = "dict";
-
- public static final float BOOST_WEIGHT = 10f;
- // 相似查询返回的最大数据量
- public static final Integer MAX_RESULT_WINDOWS_SIZE = 10000;
- public static final Integer MAX_SCROLL_SIZE = 5000;
- public static final Integer MAX_SCORLL_SIZE_CL = 2000;
- public static final Long MAX_SEARCH_TIME_MINUTES = 2L;
- public static final String SUBJECT_CLUSTER_INDEX_PREFIX = "mf_subject";
- public static final String ES_KEY = "es";
- public static final String ES_NORMAL_KEY = "es_normal";
- public static final String CACHE_KEY = "cache";
- public static final int BULK_SIZE = 100;
- public static final String INDEX_TYPE = "docs";
- public static final String bigramPostfix = ".shingles";
- public static final String bigramKey = "bigram";
- public static final String filterTypeKey = "filter_type";
- public static final String queryTypeKey = "query_type";
- public static final Double aggOffset = 16 * 6
- * 60 * 1000d;
- public static final Double ONE_DAY = 24 * 60 * 60 * 1000d;
- public static final Double ONE_HOUR = 60 * 60 * 1000d;
-
- public static final String SENTIMENT = "sentiment";
-
- public static final String SUMMARY = "summary";
-
- /**
- * *
- *
- * @FIXED 2017.11.13 increase the runtime,from 3 min to 6 min
- */
- public static final long THREAD_TIME = 15 * 60 * 1000L;
-
- public static String DESC = "desc";
-
- public static String ASC = "asc";
-
- /**
- * pubTime时间由远及近
- */
- public static String DEFUALT_SORT = "createdTimeDesc";
-
- /**
- * pubTime时间由远及近
- */
- public static String PUBTIME_ASC_SORT_FLAG = "timeAsc";
-
-
- /**
- * pubTime时间由近及远
- */
- public static String PUBTIME_DESC_SORT_FLAG = "timeDesc";
-
- /**
- * createTime时间由近及远
- */
- public static String CREATE_TIME_DESC_SORT_FLAG = "createdTimeDesc";
-
- /**
- * createTime时间由远及近
- */
- public static String CREATE_TIME_ASC_SORT_FLAG = "createdTimeAsc";
-
- /**
- * 重要度排序
- */
- public static String SCORE_DESC_SORT_FLAG = "scoreDesc";
-
- /**
- * Alexa
- */
- public static String ALEXA_ASC_SORT_FLAG = "alexaAsc";
- /***
- *
- */
- public static Float ALEXA_DEFAULT_SCORE = 1000000.00F;
- /**
- * 转载量排序
- */
- public static String FOLDER_NUMBER_DESC_SORT_FLAG = "folderNumberDescSortFlag";
- /**
- * 综合排序:1:转载量排序;2:转载量一样按照得分排序;3:随后按照pubTime时间倒序
- */
- public static String REPORT_SORT_FLAG = "reportSortFlag";
-
- public static String SCORE = "_score";
- public static String SCORE_FROM_SOURCE = "score";
-
-
- /**
- * 关键词前缀
- */
- public static final String TAG_START = "";
- /**
- * 关键词后缀
- */
- public static final String TAG_END = "";
-
- /**
- * 标题关键词分隔符
- */
- public static final String TITLE_KEYWORDS_SPLIT_SYSBOL = ",";
- /**
- * 相关度等级词分隔符
- */
- public static final String RELATEDWORDS_HIGH = "high";
- public static final String RELATEDWORDS_MEDIUM = "medium";
- public static final String RELATEDWORDS_LOW = "low";
- /**
- * 相关度分级
- */
- public static final float RELATEDRATE_MIN = 0.00F;
- public static final float RELATEDRATE_MAX = 1.00F;
- public static final int RELATEDRATE_MIN_SEARCH_NUMBER = 10;
- /**
- * 排除关键词分隔符
- */
- public static final String EXCLUDE_KEYWORDS_SPLIT_SYSBOL = ",";
- public static final String EXCLUDE_KEYWORDS_SPLIT_CN_SYSBOL = ",";
-
-
- /**
- * 最小匹配度
- */
- public static final Integer MINI_NUMBER_SHOULD_MATCH = 1;
- /**
- * 最大摘要长度
- */
- public static final Integer MAX_FRAGMENT_SIZE = 10000;
- /**
- * 数据监控页面摘要长度
- */
- public static final Integer MONITOR_FRAGMENT_SIZE = 110;
- // 报告摘要长度
- public static final Integer Report_FRAGMENT_SIZE = 100;
- public static final Integer Foreign_Report_FRAGMENT_SIZE = 200;
- /**
- * 数据监控页面标题高亮长度
- */
- public static final Integer MONITOR_FRAGMENT_TITLE_SIZE = 50;
- /**
- * 数据导出查询大集群分页查询,每次最大查询量
- */
- public static final Integer EXPORT_SCROLL_SIZE = 1000;
- /**
- * 自定义长度
- */
- // title 长度
- public static final Integer MONITOR_FRAGMENT_USER_DEFINED_TITLE_SIZE = 30;
- // 系统报告自定义长度 40
- public static final Integer SYS_REPORT_FRAGMENT_USER_DEFINED_TITLE_SIZE = 40;
- public static final Integer FOREIGN_SYS_REPORT_FRAGMENT_USER_DEFINED_TITLE_SIZE = 80;
-
- public static final Integer MAX_FRAGMENT_USER_DEFINED_TITLE_SIZE = 50;
-
- // 预警报告的摘要长度
- public static final Integer MONITOR_FRAGMENT_USER_DEFINED_DIGEST_SIZE = 150;
- public static final String DEFAULT_JOIN_SYMBOL = ".";
- /**
- * 主体情感默认名称
- */
- public static final String DEFAULT_ENTRY_NAME = "default";
- public static final String EMOTION_ENTRY = "emotionEntry";
- public static final String EMOTION_VALUE = "emotionValue";
- public static final String EMOTION_NAME = "entryName";
- public static final String OFF_SET = "offSet";
- public static final String MINIMUM_SHOULD_MATCH = "1";
- /*
- * docType(news\bbs...):int 类型常量
- */
- public static final int DOCTYPENEWS = 0;
- public static final int DOCTYPEBBS = 1;
- /*
- * 组合查询语句key值类型
- */
- public static final String KEYTYPEOPERA = "operator";
- public static final String KEYTYPEWORD = "word";
- /*
- * 数据来源渠道 (hl \ bfd)
- */
- public static final String CHANNELHL = "HL";
- public static final String CHANNELBFD = "BFD";
- /*
- *1:type:content、title、all
- *2: text:words(目前要是关键词列表)
- */
- public static final String SEARCH_SCOPE_TYPE_TITLE = "title";
- public static final String ICON = "icon";
-
- public static final String SEARCH_SCOPE_TYPE_CONTENT = "content";
-
- //标题+正文
- public static final String SEARCH_SCOPE_TYPE_TITLE_ALL = "title_all";
-
- public static final String SEARCH_SCOPE_TYPE_AUTHOR = "author";
-
- public static final String SEARCH_SCOPE_TYPE_FORWARD = "forwardContent";
-
- public static final String SEARCH_SCOPE_TYPE_TITLE_ORIGIN = "title_origin";
-
- public static final String SEARCH_SCOPE_TYPE_CONTENT_ORIGIN = "content_origin";
-
- public static final String SEARCH_SCOPE_TYPE_ALL = "all";
-
- public static final String SEARCH_SCOPE_WECHAT_SUBSCRIPTION = "wechat_subscription";
-
- /**
- * SQ 4.0新增 UGC EGC PGC
- */
- public static final String SEARCH_SCOPE_UGC = "ugc";
-
- public static final String SEARCH_SCOPE_EGC = "egc";
-
- public static final String SEARCH_SCOPE_PGC = "pgc";
- /**
- * 关键词
- */
- public static final String SEARCH_SCOPE_TYPE_KEYWORDS = "keyWords";
-
- public static final String REL_TYPE = "relType";
-
- public static final String PRIMARY_POST = "primaryPost";
-
- public static final String PRIMARY = "primary";
-
- public static final String PT_PRIMARY = "ptPrimary";
-
- public static final String DT_PRIMARY = "dtPrimary";
-
- public static final String PT_ALL = "ptAll.keyword";
-
- public static final String DT_ALL = "dtAll.keyword";
-
- public static final String DATA_COUNT = "dataCount";
-
- /**
- * 回溯开始时间
- */
- public static final String CON_START_TIME = "conStartTime";
- /**
- * 回溯结束时间
- */
- public static final String CON_END_TIME = "conEndTime";
- public static final String SUBJECT_ID = "subjectId";
- /*
- *1:非敏感 positive
- *2:敏感 negative
- */
- public static final String COMMON_TAG = "common";
- public static final String NEGATIVE_TAG = "negative";
- public static final String NEUTER_TAG = "neuter";
-
- public static final Integer COMMON_TAG_VALUE = 1;
- public static final Integer NEGATIVE_TAG_VALUE = 3;
- public static final Integer NEUTER_TAG_VALUE = 2;
-
- public static final String COMMON_CN_TAG = "正面";
- public static final String NEGATIVE_CN_TAG = "负面";
- public static final String NEUTER_CN_TAG = "中性";
-
- public static final String AVAILABILITY = "availability";
-
- public static final List SENTIMENT_LIST = new ArrayList();
- public static String URL_HASH = "urlHash";
-
- static {
- SENTIMENT_LIST.add(COMMON_TAG);
- SENTIMENT_LIST.add(NEGATIVE_TAG);
- SENTIMENT_LIST.add(NEUTER_TAG);
- }
-
- // 正面 [0.8-1]
- // 中性(0.2-0.8)
- // 负面 [0-0.2]
- public static final Double SENTIMENTAL_MID = 0.8;
- public static final Double SENTIMENTAL_THRESHOLD = 0.2;
- public static final Double SENTIMENTAL_MIN = 0.0;
- public static final Double SENTIMENTAL_MAX = 1.0;
- public static final String OPERATOR_OR = "or";
- public static final String OPERATOR_EQUAL = "=";
- public static final String OPERATOR_SPACE = " ";
- public static final String OPERATOR_ALL = "*:*";
- public static final String OPERATOR_CONNENT_ALL = ":";
- public static final String OPERATOR_SINGLE_ALL = "*";
-
-
- /*
- *define es mapping fields
- */
- public static String PUBTIME = "pubTime";
- public static String CHANNEL_HYLANDA = "hylanda";
-
- public static String PUBTIME_STR = "pubTimeStr";
-
- public static String CREATETIME = "createTime";
- public static String CREATETIMESTR = "createTimeStr";
- public static String CREATETIMEDAY = "createDay";
-
- public static String PUTTIME = "putTime";
- public static String CRAWLTIME = "crawlTime";
- public static String ORIGINAL_SOURCE = "originalSource";
- public static String CONTENT_SIMHASH = "contentSimHash";
- public static String QUOTE_COUNT = "quoteCount";
- /**
- * 内容
- */
- public static String CONTENT_TAG = "contentTag";
- public static String DT_EN_2 = "dtEn2";
- public static String DT_EN_1 = "dtEn1";
- public static String COMMENTS_COUNT = "commentsCount";
- public static String PAGE_COMMENTS_COUNT = "pageCommentCount";
- public static String ATTITUDES_COUNT = "attitudesCount";
- public static String TITLE_SIMHASH = "titleSimHash";
- public static String ORIGINAL_CONTENT_SIMHASH = "originalContentSimHash";
- public static String ORIGINAL_TITLE_SIMHASH = "originalTitleSimHash";
- public static String DOC_TYPE = "docType";
- public static String CHINA_DOC_TYPE = "chinaDocType";
- public static String SYS_SENTIMENT_TAG = "sysSentimentTag";
- public static final String TITLE = "title";
- public static final String USER_LEVEL_NAME = "userLevelName";
- public static String SOURCE = "source";
-
- //微信专属字段,内容固定 value="微信"
- public static String WEIXIN_SOURCE = "weixinSource";
- public static String CHANNEL = "channel";
- public static final String CONTENT = "content";
- public static final String SYS_SENTIMENT = "sysSentiment";
- /**
- * DT情感标签
- */
- public static final String DT_SENTIMENT = "dtSentiment";
- /**
- * PT情感标签
- */
- public static final String PT_SENTIMENT = "ptSentiment";
-
- public static String URL = "url";
- public static String DOC_ID = "docId";
- public static String _ID = "_id_";
- public static String SYS_ABSTRACT = "sysAbstract";
- public static String SYS_KEYWORDS = "sysKeywords";
- // 所有热词
- public static String HL_KEYWORDS = "hlKeywords";
- public static String ALL_KEYWORDS = "allKeywords";
- public static String COMMENT_KEY_WORDS = "commentKeywords";
- public static String OPTIONS = "opinions";
- // 地名:places
- public static String PLACES = "places";
- // 微博话题:hashTag
- public static String HASH_TAG = "hashTag";
- public static String PICTURE_LIST = "pictureList";
-
- // 表情:expression
- public static String EXPRESSION = "expression";
- // 评价:opinions
- public static String OPINIONS = "opinions";
-
- public static String WEIBO_AUTHOR = "author";
- public static String WEIBO_USER_ID = "userId";
- public static String AUTHOR = "author";
- public static String WEIXIN_NAME = "weixinName";
- public static final String CHANNEL_WEIXIN = "weixin";
- public static final String HL_DOCID = "hylandaDocId";
- public static final String LANGUAGE = "language";
-
-
- /**
- * _all字段
- */
- public static String _ALL = "_all";
-
- /**
- * 口碑
- */
- public static final String SHOP_MD5 = "shopUrlHash";
- public static final String HAS_PICTURE = "isCommentImg";
- public static final String HAS_CONTENT = "isComment";
- public static final String CUSTOMER_LEVEL = "customerLevel";
- public static final String COMMENT_STAR_LEVEL = "starLevel";
- public static final String SHOP_NAME = "shopName";
- public static final String NICKNAME = "nickName";
- public static final String COMMENT_SEARCH_ALL = "comment_all";
- public static final String USER_AVATAR = "userAvatar";
- public static final String UNIQUE_ID = "uniqueId";
- public static final String SHOP_URL = "shopUrl";
- public static final String USER_URL = "userUrl";
- public static final String CONTRIBUTION = "contribution";
- public static final String SCORE_FACTOR = "scoreFactor";
- public static final String FACTOR_KEY = "factorKey";
- public static final String FACTOR_VALUE = "factorValue";
- public static final String CONTENT_SIZE = "commentLength";
- public static final String SHOP_LABEL = "label";
- public static final String SHOP_CITY = "shopCity";
- public static final String SHOP_PROVINCE = "shopProvince";
- public static final String USER_CITY = "userCity";
- public static final String COUNTRY = "country";
- public static final String USER_PROVINCE = "userProvince";
- public static final String SHOP_LABELS = "shopLabels";
- public static final String COMMENT_IMG = "commentImg";
-
- /**
- * 用户头像
- */
- public static String WEIBO_AVATOR = "avatar";
- public static final String BBS = "bbs";//论坛
- public static final String WEI_BO = "weibo";//微博
- public static final String WEI_XIN = "weixin";//微信
- public static final String CH_WEI_XIN = "微信";
- public static final String BLOG = "blog";//博客
- public static final String NEWS = "news";//网站
- public static final String TIEBA = "tieba";//贴吧
- public static final String MEDIA = "media";//报刊
- public static final String VIDEO = "video";//视频
- public static final String WENDA = "wenda";//问答
- public static final String SHEJIAO = "shejiao";// 社交
- public static final String APP = "app";
- public static String WEIBO_DEFAULT_AVATOR = "http://tva1.sinaimg.cn/default/images/default_avatar_male_180.gif";
-
- public static final String BAIDUKOUBEI = "baidukoubei";
- public static final String DIANPING = "dianping";
- public static final String FANGTIANXIA = "fangtianxia";
- public static final String BAIDUTIEBA = "baidutieba";
- public static final String SOUHU = "souhu";
- public static final String XIECHENG = "xiecheng";
- public static final String AUTOHOME = "autohome";
- public static final String TMALL = "tmall";
- public static final String YILONG = "yilong";
- // 电商
- public static final String ITEM = "item";
- public static final String ITEM_NAME = "itemName";
- public static final String ZIXUN = "zixun";
- public static final String FOREIGN_NEWS = "foreign_news";
- public static final String FOREIGN_MEDIA = "foreign_media";
- //电商星级分数
- public static final String COMMENT_SCORE = "commentScore";
- // 电商数据没有星级时返回-1
- public static final Double COMMENT_SCORE_NULL = -1.0;
- public static final String EN_SOURCE = "enSource";
-
- //内容识别
- public static final String NORMAL = "normal";
- public static final String MARKETING = "marketing";
-
- public static final String AVATAR = "avatar";
- public static final String COMMENT_ID = "commentId";
-
-
- /**
- * 微博信息 added by Eric 2016-10-28 15:03:31
- */
- public static String WEIBO_USER_SEX = "sex";// 性别 female \ male
- public static String WEIBO_USER_SEX_FEMALE = "female";// 性别 female
- public static String WEIBO_USER_SEX_MALE = "male";
-
- public static String WEIBO_USER_FRIENDS_COUNT = "friendsCount";// 关注数
- public static String WEIBO_USER_FANS_COUNT = "fansCount";// 粉丝数
- public static String WEIBO_ATTITUDES_COUNT = "attitudesCount";// 赞数
- public static String WEIBO_COMMENTS_COUNT = "commentsCount";// 评论数
- public static String WEIBO_QUOTE_COUNT = "quoteCount";// 转发数
- public static String WEIBO_COUNT = "weiboCount";//用户转发微博数量
- public static String WEIBO_LOCATION = "location";//用户转发微博数量
- public static String WEIBO_USER_COUNTRY = "country";// 国家
- public static String WEIBO_USER_PROVINCE = "province";// 省份
- public static String WEIBO_USER_CITY = "city";// 城市 verified
- public static String WEIBO_USER_AREA = "area";
- public static String WEIBO_MUNICIPALITY_BEIJING = "北京";
- public static String WEIBO_MUNICIPALITY_TIANJIN = "天津";
- public static String WEIBO_MUNICIPALITY_CHONGQING = "重庆";
- public static String WEIBO_MUNICIPALITY_SHANGHAI = "上海";
-
- public static String WEIBO_USER_VERIFIED = "verified";// 认证类型
- public static String WEIBO_USER_NOT_VERIFIED_VALUE = "0";// 非认证类型
- public static String WEIBO_ORIGINAL_URL = "originalUrl";// 上层url
- public static String WEIBO_EXPRESSION = "expression";// 表情图
- public static String WEIBO_EXPRESSION_TEXT = "expressionText";// 表情文本
- public static String WEIBO_FORWARD_URL = "forwardUrl";// 原文url
- public static String FORWARD_USER_URL = "forwardUserUrl";// 原文url
- public static String WEIBO_USER_TYPE = "userType";
- public static String WEIBO_POST_SOURCE = "postSource";
- public static String WEIBO_LEVEL = "level";
- public static String WEIBO_REPLY_COMMENT = "replycomment";
- public static String PRODUCT_ID = "productId";
-
-
- /**
- * 微博转发信息
- */
- public static final String FORWARD_CONTENT = "forwardContent"; // 转发原文内容
- public static final String FORWARD_RELEASE_DATE = "forwardReleaseDate"; // 转发原文发布时间
- public static final String FORWARD_AUTHOR = "forwardAuthor"; // 转发原文作者昵称
-
- public static final String FORWARD_QUOTE_COUNT = "forwardQuoteCount";
- public static final String FORWARD_POST_SOURCE = "forwardPostSource";
- public static final String FORWARD_COMMENT_COUNT = "forwardCommentsCount";
- public static final String FORWARD_ATTITUDES_COUNT = "forwardAttitudesCount";
- public static final String FORWARD_URL = "forwardUrl";
- public static final String FORWARD_AVATAR = "forwardAvatar";
- public static final String FORWARD_USER_TYPE = "forwardUserType";
- public static final String FORWARD_PUBTIME = "forwardPubTime";
-
-
- public static final String FORWARD_FLAG = "forwardFlag"; // 转发、回帖标志 1(转发、回帖) 0(原贴)
-
- public static final int REGULAR_PUBTIME_QUERY = 0;
-
- public static final int REGULAR_PUBTIME_AND_CREATETIME_QUERY = 1;
-
-
- public static final List TYPE_LIST = new ArrayList<>();
-
- public static final List BACKEND_ANALYSIS_LIST = new ArrayList<>();
-
- public static final List BACKEND_AVAILABILITY_LIST = new ArrayList<>();
-
- static {
- BACKEND_ANALYSIS_LIST.add(1L);
- BACKEND_ANALYSIS_LIST.add(2L);
- BACKEND_AVAILABILITY_LIST.add(0L);
- BACKEND_AVAILABILITY_LIST.add(1L);
- }
-
- static {
- TYPE_LIST.add(BBS);
- TYPE_LIST.add(WEI_XIN);
- TYPE_LIST.add(BLOG);
- TYPE_LIST.add(WEI_BO);
- TYPE_LIST.add(TIEBA);
- TYPE_LIST.add(MEDIA);
- TYPE_LIST.add(VIDEO);
- TYPE_LIST.add(WENDA);
- TYPE_LIST.add(ZIXUN);
- TYPE_LIST.add(SHEJIAO);
-
- }
-
- public static final List APP_TYPE_LIST = new ArrayList();
-
- static {
- APP_TYPE_LIST.add(BBS);
- APP_TYPE_LIST.add(WEI_XIN);
- APP_TYPE_LIST.add(BLOG);
- APP_TYPE_LIST.add(WEI_BO);
- APP_TYPE_LIST.add(ZIXUN);
- APP_TYPE_LIST.add(TIEBA);
- APP_TYPE_LIST.add(MEDIA);
- APP_TYPE_LIST.add(VIDEO);
- APP_TYPE_LIST.add(WENDA);
- APP_TYPE_LIST.add(FOREIGN_MEDIA);
- APP_TYPE_LIST.add(FOREIGN_NEWS);
- }
-
- /**
- * 口碑评价对应渠道类型
- */
- public static final List ITEM_TYPE_LIST = new ArrayList();
-
- private static final String MEITUAN = "meituan";
-
- private static final String DIAPING = "dianping";
-
- public static final String ID = "id";
-
- public static final String COMMENTS = "comments";
-
- public static final String REPOSTS = "reposts";
-
- public static final String ATTITUDES = "attitudes";
-
-// private static final String JD = "jd";
-//
-// private static final String TAOBAO = "taobao";
-//
-// private static final String TMALL = "tmall";
-
- static {
- ITEM_TYPE_LIST.add(MEITUAN);
- ITEM_TYPE_LIST.add(DIAPING);
-// ITEM_TYPE_LIST.add(JD);
-// ITEM_TYPE_LIST.add(TAOBAO);
-// ITEM_TYPE_LIST.add(TMALL);
-
- }
-
- public static final Map SORT_FLAG_MAPPING_MYSQL_SORT = new HashMap<>();
-
- static {
- SORT_FLAG_MAPPING_MYSQL_SORT.put(PUBTIME_DESC_SORT_FLAG, "pub_time desc");
- SORT_FLAG_MAPPING_MYSQL_SORT.put(PUBTIME_ASC_SORT_FLAG, "pub_time asc");
- SORT_FLAG_MAPPING_MYSQL_SORT.put(FOLDER_NUMBER_DESC_SORT_FLAG, "similarity_number desc");
- SORT_FLAG_MAPPING_MYSQL_SORT.put(CREATE_TIME_DESC_SORT_FLAG, "create_time desc");
- SORT_FLAG_MAPPING_MYSQL_SORT.put(CREATE_TIME_ASC_SORT_FLAG, "create_time asc");
- }
-
-
- public static final String[] FIELD_LIST_MONITOR_NORMAL = EsQueryConditionUtils.getIncludeQueryField(ESConstant.CONTENT_SIMHASH
- , ESConstant.TITLE_SIMHASH
- , ESConstant.TITLE
- , ESConstant.SOURCE
- , ESConstant.PUBTIME
- , ESConstant.DOC_TYPE
- , ESConstant.URL
- , ESConstant.SYS_SENTIMENT
- , ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_NAME
- , ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_VALUE
- , ESConstant.DOC_ID
- , ESConstant.SYS_ABSTRACT
- , ESConstant.WEIBO_AUTHOR
- , ESConstant.WEIBO_AVATOR
- , ESConstant.WEIBO_USER_SEX
- , ESConstant.WEIBO_USER_FRIENDS_COUNT
- , ESConstant.WEIBO_USER_FANS_COUNT
- , ESConstant.WEIBO_ATTITUDES_COUNT
- , ESConstant.WEIBO_COMMENTS_COUNT
- , ESConstant.WEIBO_QUOTE_COUNT
- , ESConstant.WEIBO_USER_TYPE
- , ESConstant.WEIBO_USER_COUNTRY
- , ESConstant.WEIBO_USER_PROVINCE
- , ESConstant.WEIBO_USER_CITY
- , ESConstant.WEIBO_USER_AREA
- , ESConstant.WEIBO_USER_VERIFIED
- , ESConstant.FORWARD_AUTHOR
- , ESConstant.FORWARD_RELEASE_DATE
- , ESConstant.FORWARD_CONTENT
- , ESConstant.FORWARD_FLAG
- , ESConstant.COMMENT_SCORE
- , ESConstant.ATTITUDES_COUNT
- , ESConstant.FORWARD_QUOTE_COUNT
- , ESConstant.FORWARD_ATTITUDES_COUNT
- , ESConstant.FORWARD_COMMENT_COUNT
- , ESConstant.FORWARD_PUBTIME
- , ESConstant.COMMENT_ID
- , ESConstant.QUOTE_COUNT
- , ESConstant.COMMENTS_COUNT
- , FORWARD_POST_SOURCE
- , WEIBO_POST_SOURCE
- , FORWARD_URL
- , FORWARD_AVATAR
- , FORWARD_USER_TYPE
- , ESConstant.ORIGINAL_SOURCE
- , ESConstant.FORWARD_USER_URL
- , ESConstant.CONTENT_TAG);
-
- public static final String[] FIELD_LIST_CRAWLEC_NORMAL = EsQueryConditionUtils.getIncludeQueryField(ESConstant.CONTENT_SIMHASH
- , ESConstant.TITLE_SIMHASH
- , ESConstant.ITEM_NAME
- , ESConstant.TITLE
- , ESConstant.SOURCE
- , ESConstant.PUBTIME
- , ESConstant.DOC_TYPE
- , ESConstant.URL
- , ESConstant.SYS_SENTIMENT
- , ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_NAME
- , ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_VALUE
- , ESConstant.DOC_ID
- , ESConstant.SYS_ABSTRACT
- , ESConstant.WEIBO_AUTHOR
- , ESConstant.WEIBO_AVATOR
- , ESConstant.WEIBO_USER_SEX
- , ESConstant.WEIBO_USER_FRIENDS_COUNT
- , ESConstant.WEIBO_USER_FANS_COUNT
- , ESConstant.WEIBO_ATTITUDES_COUNT
- , ESConstant.WEIBO_COMMENTS_COUNT
- , ESConstant.WEIBO_QUOTE_COUNT
- , ESConstant.WEIBO_USER_TYPE
- , ESConstant.WEIBO_USER_COUNTRY
- , ESConstant.WEIBO_USER_PROVINCE
- , ESConstant.WEIBO_USER_CITY
- , ESConstant.WEIBO_USER_AREA
- , ESConstant.WEIBO_USER_VERIFIED
- , ESConstant.FORWARD_AUTHOR
- , ESConstant.FORWARD_RELEASE_DATE
- , ESConstant.FORWARD_CONTENT
- , ESConstant.FORWARD_FLAG
- , ESConstant.COMMENT_SCORE
- , ESConstant.ATTITUDES_COUNT
- , ESConstant.FORWARD_QUOTE_COUNT
- , ESConstant.FORWARD_ATTITUDES_COUNT
- , ESConstant.FORWARD_COMMENT_COUNT
- , ESConstant.FORWARD_PUBTIME
- , ESConstant.COMMENT_ID
- , ESConstant.QUOTE_COUNT
- , ESConstant.COMMENTS_COUNT
- , FORWARD_POST_SOURCE
- , WEIBO_POST_SOURCE
- , FORWARD_URL
- , FORWARD_AVATAR
- , FORWARD_USER_TYPE
- , ESConstant.ORIGINAL_SOURCE
- , ESConstant.FORWARD_USER_URL
- , ESConstant.CONTENT_TAG);
-
- public static final String[] FIELD_LIST_MONITOR = {
- ESConstant.PUBTIME,
- ESConstant.TITLE_SIMHASH,
- ESConstant.SCORE_FROM_SOURCE,
- ESConstant.DOC_ID,
- ESConstant.WEIBO_USER_PROVINCE,
- ESConstant.SOURCE,
- ESConstant.DOC_TYPE,
- ESConstant.SYS_SENTIMENT,
- ESConstant.AUTHOR,
- ESConstant.QUOTE_COUNT,
- ESConstant.WEIBO_USER_TYPE,
- ESConstant.COMMENTS_COUNT,
- ESConstant.CONTENT_TAG,
- ESConstant.AVATAR};
-
- public static final String[] FIELD_LIST_CRAWLEC = {
- ESConstant.PUBTIME,
- ESConstant._ID,
- ESConstant.SCORE_FROM_SOURCE,
- ESConstant.DOC_ID,
- ESConstant.SOURCE,
- ESConstant.DOC_TYPE,
- ESConstant.SYS_SENTIMENT,
- ESConstant.AUTHOR,
- ESConstant.CONTENT_TAG,
- ESConstant.AVATAR,
- ESConstant.ITEM_NAME};
-
-
- public static final String[] FIELD_LIST_OTHER = {
- ESConstant.PUBTIME,
- ESConstant._ID,
- ESConstant.SCORE_FROM_SOURCE,
- ESConstant.DOC_ID,
- ESConstant.WEIBO_USER_PROVINCE,
- ESConstant.SOURCE,
- ESConstant.DOC_TYPE,
- ESConstant.SYS_SENTIMENT,
- ESConstant.AUTHOR,
- ESConstant.QUOTE_COUNT,
- ESConstant.WEIBO_USER_TYPE,
- ESConstant.COMMENTS_COUNT,
- ESConstant.CONTENT_TAG,
- ESConstant.AVATAR,
- ESConstant.ITEM_NAME};
-
-
- public static final String[] FIELD_LIST_REPORT = {
- ESConstant.PUBTIME,
- ESConstant.TITLE_SIMHASH,
- ESConstant.SCORE_FROM_SOURCE,
- ESConstant.DOC_ID,
- ESConstant.DOC_TYPE,
- ESConstant.SOURCE,
- ESConstant.URL,
- ESConstant.SYS_SENTIMENT,
- ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_NAME,
- ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_VALUE};
-
- public static final String[] FIELD_LIST_CLOUD = {
- ESConstant.HL_KEYWORDS,
- ESConstant.PUBTIME,
- ESConstant.DOC_ID,
- ESConstant.DOC_TYPE,
- ESConstant.SOURCE,
- ESConstant.SYS_SENTIMENT};
-
-
- public static final String[] FIELD_LIST_ANALYSIS =
- EsQueryConditionUtils.getIncludeQueryField(
- ESConstant.PUBTIME
- , ESConstant.TITLE_SIMHASH
- , ESConstant.SCORE_FROM_SOURCE
- , ESConstant.DOC_ID
- , ESConstant.DOC_TYPE
- , ESConstant.HL_KEYWORDS
- , ESConstant.SOURCE
- , ESConstant.SYS_SENTIMENT
- , ESConstant.WEIBO_AUTHOR
- , ESConstant.WEIBO_USER_SEX
- , ESConstant.WEIBO_USER_FRIENDS_COUNT
- , ESConstant.WEIBO_USER_FANS_COUNT
- , ESConstant.WEIBO_USER_COUNTRY
- , ESConstant.WEIBO_USER_PROVINCE
- , ESConstant.WEIBO_USER_CITY
- , ESConstant.WEIBO_USER_VERIFIED
- , ESConstant.CONTENT_TAG
- , ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_NAME
- , ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_VALUE
- );
-
- /**
- * 话题对比分析
- */
- public static final String[] TOPIC_COMPARE_ANALYSIS =
- EsQueryConditionUtils.getIncludeQueryField(
- ESConstant.PUBTIME
- , ESConstant.TITLE_SIMHASH
- , ESConstant.DOC_ID
- , ESConstant.DOC_TYPE,
- ESConstant.SYS_SENTIMENT
- );
- public static final String[] FIELD_CLOUD_ANALYSIS = EsQueryConditionUtils.getIncludeQueryField(
- // 所有
- ESConstant.HL_KEYWORDS
- // 地名
- , ESConstant.PLACES
- // 话题
- , ESConstant.HASH_TAG
- // 评价
- , ESConstant.OPINIONS
- // 表情
- , ESConstant.EXPRESSION
- , ESConstant.DOC_ID
- , ESConstant.SYS_SENTIMENT
- , ESConstant.CONTENT_TAG
- , ESConstant.PUBTIME
- , ESConstant.DOC_TYPE
- );
-
- public static final String[] FIELD_CLOUD_ANALYSIS_CRAWL = EsQueryConditionUtils.getIncludeQueryField(
- // 所有
- ESConstant.HL_KEYWORDS
- // 地名
- , ESConstant.PLACES
- // 话题
- , ESConstant.HASH_TAG
- // 评价
- , ESConstant.OPINIONS
- // 表情
- , ESConstant.EXPRESSION
- , ESConstant.DOC_ID
- , ESConstant.SYS_SENTIMENT
- , ESConstant.CONTENT_TAG
- , ESConstant.PUBTIME
- , ESConstant.DOC_TYPE
- );
-
- public static final String[] FILED_SIMILAR_QUERY = EsQueryConditionUtils.getIncludeQueryField(ESConstant.CONTENT_SIMHASH
- , ESConstant.TITLE_SIMHASH
- , ESConstant.TITLE
- , ESConstant.SOURCE
- , ESConstant.ORIGINAL_SOURCE
- , ESConstant.PUBTIME
- , ESConstant.DOC_TYPE
- , ESConstant.QUOTE_COUNT
- , ESConstant.COMMENTS_COUNT
- , ESConstant.URL
- , ESConstant.CONTENT_TAG
- , ESConstant.SYS_SENTIMENT
- , ESConstant.CONTENT
- , ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_NAME
- , ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_VALUE
- , ESConstant.DOC_ID
- , ESConstant.WEIBO_AUTHOR
- , ESConstant.SYS_ABSTRACT
- , ESConstant.TITLE_SIMHASH
- , ESConstant.WEIBO_USER_COUNTRY
- , ESConstant.WEIBO_USER_PROVINCE
- , WEIBO_POST_SOURCE
- , WEIBO_USER_TYPE
- , QUOTE_COUNT
- , ATTITUDES_COUNT
- , COMMENTS_COUNT
- , COMMENT_ID
- , COMMENT_SCORE
- , ESConstant.WEIBO_USER_CITY
- , ESConstant.WEIBO_USER_AREA
- , ESConstant.WEIBO_AVATOR);
-
- public static final String[] FIELD_LIST_ANALYSIS_TREND =
- EsQueryConditionUtils.getIncludeQueryField(ESConstant.PUBTIME
- , ESConstant.DOC_ID
- , ESConstant.DOC_TYPE
- , ESConstant.SYS_SENTIMENT
- , ESConstant.CRAWLTIME
- );
-
- public static final String[] FIELD_LIST_ANALYSIS_WEIBO =
- EsQueryConditionUtils.getIncludeQueryField(ESConstant.PUBTIME
- , ESConstant.TITLE_SIMHASH
- , ESConstant.SCORE_FROM_SOURCE
- , ESConstant.DOC_ID
- , ESConstant.DOC_TYPE
- , ESConstant.SOURCE
- , ESConstant.SYS_SENTIMENT
- , ESConstant.WEIBO_AUTHOR
- , ESConstant.WEIBO_USER_SEX
- , ESConstant.WEIBO_USER_FRIENDS_COUNT
- , ESConstant.WEIBO_USER_FANS_COUNT
- , ESConstant.WEIBO_USER_COUNTRY
- , ESConstant.WEIBO_USER_PROVINCE
- , ESConstant.WEIBO_USER_CITY
- , ESConstant.WEIBO_USER_VERIFIED
- , ESConstant.WEIBO_POST_SOURCE
- , ESConstant.FORWARD_QUOTE_COUNT
- , ESConstant.FORWARD_COMMENT_COUNT
- , ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_NAME
- , ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_VALUE
- , ESConstant.CONTENT_TAG
- , ESConstant.COMMENT_ID
- );
-
- public static final String[] FIELD_LIST_ANALYSIS_WEIBO_USER =
- EsQueryConditionUtils.getIncludeQueryField(ESConstant.PUBTIME
- , ESConstant.DOC_ID
- , ESConstant.TITLE_SIMHASH
- , ESConstant.WEIBO_USER_SEX
- , ESConstant.WEIBO_USER_FRIENDS_COUNT
- , ESConstant.WEIBO_USER_FANS_COUNT
- , ESConstant.WEIBO_USER_COUNTRY
- , ESConstant.WEIBO_USER_PROVINCE
- , ESConstant.WEIBO_USER_CITY
- , ESConstant.WEIBO_AVATOR
- );
-
- public static final String[] FIELD_LIST_EMOTION = EsQueryConditionUtils.getIncludeQueryField(ESConstant.SYS_SENTIMENT
- , ESConstant.TITLE
- , ESConstant.TITLE_SIMHASH
- , ESConstant.DOC_ID
- , ESConstant.DOC_TYPE
- , ESConstant.URL
- , ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_NAME
- , ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_VALUE);
- public static final int MAX_UPDATE_NUMBER = 1000;
- /**
- * updated the settings;
- * from default 100 to 1000
- */
- public static final int MAX_INNER_RESULT_WINDOW = 1000;
-
- public static final String[] FIELD_LIST_DOWNLOAD_SMALL = {ESConstant.PUBTIME, ESConstant.TITLE_SIMHASH, ESConstant.SCORE_FROM_SOURCE, ESConstant.DOC_ID, ESConstant.WEIBO_USER_PROVINCE, ESConstant.WEIBO_USER_CITY};
- public static String[] FIELD_LIST_DOWNLOAD = EsQueryConditionUtils.getIncludeQueryField(
- ESConstant.CONTENT_SIMHASH
- , ESConstant.TITLE_SIMHASH
- , ESConstant.TITLE
- , ESConstant.SOURCE
- , ESConstant.PUBTIME
- , ESConstant.DOC_TYPE
- , ESConstant.URL
- , ESConstant.SYS_SENTIMENT
- , ESConstant.DOC_ID
- , ESConstant.SYS_ABSTRACT
- , ESConstant.WEIBO_AUTHOR
- , ESConstant.WEIBO_AVATOR
- , ESConstant.CONTENT_TAG
- , ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_NAME
- , ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_VALUE
- // Added by Eric 2016-10-28 15:03:31
- , ESConstant.WEIBO_USER_SEX
- , ESConstant.WEIBO_USER_FRIENDS_COUNT
- , ESConstant.WEIBO_USER_FANS_COUNT
- , ESConstant.WEIBO_ATTITUDES_COUNT
- , ESConstant.WEIBO_COMMENTS_COUNT
- , ESConstant.WEIBO_QUOTE_COUNT
- , ESConstant.WEIBO_USER_COUNTRY
- , ESConstant.WEIBO_USER_PROVINCE
- , ESConstant.WEIBO_USER_CITY
- , ESConstant.WEIBO_USER_VERIFIED
- /**@FIXED 2017.05.19 add field originalSource、content for export excel*/
- , ESConstant.ORIGINAL_SOURCE
- , ESConstant.CONTENT);
-
-
- public static HighlightBuilder HIGHLIGHT_FOR_DOWNLOAD = new HighlightBuilder().
- field(ESConstant.CONTENT).
- field(ESConstant.TITLE).
- field(ESConstant.CONTENT + ESConstant.bigramPostfix).
- field(ESConstant.TITLE + ESConstant.bigramPostfix).
- preTags(ESConstant.TAG_START).
- postTags(ESConstant.TAG_END).
- fragmentSize(ESConstant.MONITOR_FRAGMENT_SIZE);
-
-
- public static final String[] areaArray = EsQueryConditionUtils.getIncludeQueryField(
- ESConstant.WEIBO_USER_COUNTRY,
- ESConstant.WEIBO_USER_PROVINCE,
- ESConstant.WEIBO_USER_CITY,
- ESConstant.WEIBO_USER_AREA);
- public static final String[] municipalityArray = EsQueryConditionUtils.getIncludeQueryField(
- ESConstant.WEIBO_MUNICIPALITY_BEIJING,
- ESConstant.WEIBO_MUNICIPALITY_CHONGQING,
- ESConstant.WEIBO_MUNICIPALITY_SHANGHAI,
- ESConstant.WEIBO_MUNICIPALITY_TIANJIN);
-
-
- public static final String[] COMMENT_DETAILS_ARRY = {
- PUBTIME,
- DOC_TYPE,
- DOC_ID,
- AUTHOR,
- SOURCE,
- CONTENT,
- SYS_SENTIMENT,
- CONTENT_TAG,
- DT_EN_1, DT_EN_2,
- TITLE,
- USER_LEVEL_NAME,
- HL_KEYWORDS,
- COMMENT_KEY_WORDS,
- OPTIONS,
- COMMENT_SCORE,
- URL
- };
-
- public static final String BRAND = "brand";
- public static final String[] FIELD_ECOMMERCE_EXPORT = {
- PUBTIME,
- DOC_TYPE,
- DOC_ID,
- AUTHOR,
- SOURCE,
- CONTENT,
- PRODUCT_ID,
- BRAND,
- ITEM_NAME,
- SHOP_NAME,
- URL,
- "dt1",
- "dt2",
- "dt3",
- "dt4",
- "dt5",
- DT_SENTIMENT,
- "pt1",
- "pt2",
- "pt3",
- "pt4",
- "pt5",
- PT_SENTIMENT
- };
-
-
-}
diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/ESServerUtils.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/ESServerUtils.java
deleted file mode 100644
index 0123d7c..0000000
--- a/cl_search_api/src/main/java/com/bfd/mf/common/util/ESServerUtils.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Copyright (C) 2016 Baifendian Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package com.bfd.mf.common.util;
-
-import com.bfd.mf.common.service.es.ClusterService;
-import com.bfd.mf.common.web.entity.mysql.cache.Cluster;
-import com.bfd.nlp.common.util.object.TObjectUtils;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.support.IndicesOptions;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.TransportAddress;
-import org.elasticsearch.transport.client.PreBuiltTransportClient;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-
-import javax.annotation.PostConstruct;
-import java.io.Serializable;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * 区别多环境问题
- */
-@Component
-public class ESServerUtils implements Serializable {
- private static Logger logger = LoggerFactory.getLogger(ESServerUtils.class);
- private static final long serialVersionUID = 1L;
- private static Map clientByClusterId = new HashMap<>();
- @Autowired
- private ClusterService clusterService;
-
- @PostConstruct
- public void initEsServer() {
-
- Iterable iterator = clusterService.getAll();
- long startTime = System.currentTimeMillis();
- iterator.forEach((final Cluster cluster) ->
- {
- logger.info("[Initial Elastic Search] Initial ES handler, the cluster name is {}", cluster.getClusterName());
- System.setProperty("es.set.netty.runtime.available.processors", "false");
- Settings settings = Settings.builder().put("cluster.name",
- cluster.getClusterName())
- .put("client.transport.sniff", false)
- .build();
- try {
- logger.info("[Initial Elastic Search] Initial First Url= {} and First Port={} clusterName = {}",
- cluster.getIp(), cluster.getTcpPort(), cluster.getClusterName());
- TransportClient client = new PreBuiltTransportClient(settings);
- String[] ports = cluster.getTcpPort().split(",");
- int index = 0;
- for (String ip : cluster.getIp().split(",")) {
- int port = Integer.parseInt(ports[index]);
- client.addTransportAddress(new TransportAddress(InetAddress.getByName(ip), port));
- index++;
- }
- logger.info("[Initial Elastic Search] Initial ES Finished.the client is{} ", client);
- clientByClusterId.put(cluster.getId(), client);
- } catch (UnknownHostException e) {
- logger.error("UnknownHostException occurs while creating TransportClient,error message={}", e);
- System.exit(1);
- } catch (Exception ex) {
- logger.error("Exception occurs while creating TransportClient,error message={}", ex);
- System.exit(1);
- }
- logger.info("[Initial Elastic Search] Initial ES handler finish...");
- }
- );
- long endTime = System.currentTimeMillis();
- logger.info("[Initial Elastic Search] finish,the used time is {}", (endTime - startTime));
- }
-
-
- public TransportClient getClientByClusterId(Long clusterId) {
- return clientByClusterId.get(clusterId);
- }
-
- /**
- * 根据索引 List 查询指定ES
- */
- public SearchRequestBuilder buildSearchRequestBuilder(Long clusterId, List indexList) {
-
- SearchRequestBuilder indexSearch = null;
- try {
- if (TObjectUtils.isListEmpty(indexList)) {
- indexSearch = getClientByClusterId(clusterId).prepareSearch(clusterService.get(clusterId).getPrefixIndexPattern() + "_*");
- } else {
- indexSearch = getClientByClusterId(clusterId).prepareSearch((String[]) indexList.toArray(new String[indexList.size()]));
- }
- // allowNoIndices: true,不存在的索引不会抛异常;false,抛异常
- indexSearch.setIndicesOptions(IndicesOptions.fromOptions(true, true,
- true, false));
- } catch (Exception e) {
- logger.error("[ESServerUtils] buildSearchRequestBuilder: build SearchRequestBuilder error,the error message is:{}", e);
- }
- return indexSearch;
- }
-
-
-}
\ No newline at end of file
diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/EsQueryConditionUtils.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/EsQueryConditionUtils.java
deleted file mode 100644
index 74ea368..0000000
--- a/cl_search_api/src/main/java/com/bfd/mf/common/util/EsQueryConditionUtils.java
+++ /dev/null
@@ -1,20 +0,0 @@
-package com.bfd.mf.common.util;
-
-
-public class EsQueryConditionUtils {
-
-
- public static String[] getIncludeQueryField(String... includeField) {
- String[] includeFields = new String[includeField.length];
- System.arraycopy(includeField, 0, includeFields, 0, includeField.length - 1 + 1);
- return includeFields;
- }
-
-
- public static String[] getExcludeQueryField(String... excludeField) {
- String[] excludeFields = new String[excludeField.length];
- System.arraycopy(excludeField, 0, excludeFields, 0, excludeField.length - 1 + 1);
- return excludeFields;
- }
-
-}
diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/OperatorUtil.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/OperatorUtil.java
deleted file mode 100644
index 4e8d433..0000000
--- a/cl_search_api/src/main/java/com/bfd/mf/common/util/OperatorUtil.java
+++ /dev/null
@@ -1,64 +0,0 @@
-package com.bfd.mf.common.util;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-
-public class OperatorUtil {
- private static final Character[] seps = new Character[] { '&', '|', '~', '<', '《', '>', '》', '(', '(', ')', ')',
- ',', ',', '"', '“', '”' };// 分隔符,将整个表达式分隔成若干个子串进行处理
- private static final Character[] ops = new Character[] { '&', '|', '~' };// 去掉'!'
- public static final List operators = new ArrayList();
- public static final List seperators = new ArrayList();
-
- static {
- operators.addAll(Arrays.asList(ops));
- seperators.addAll(Arrays.asList(seps));
- }
-
- public static boolean isOperator(char c) {
- return operators.contains(c);
- }
-
-
- public static boolean isSeperator(char c) {
- return seperators.contains(c);
- }
-
-
- public static int priorityCompare(char op1, char op2) {
- // ()>!>&>|
- switch (op1) {
- case '!':
- case '!':
- return op2 == '(' || op2 == ')' ? -1 : 1;
- case '&':
- return op2 == '!' ? -1 : (op2 == '|' ? 1 : 0);
- case '|':
- return op2 == '|' ? 0 : -1;
- }
- return 0;
- }
-
-
- public static boolean matchChar(char c, char c2) {
- switch (c) {
- case '(':
- case '(':
- return c2 == ')' || c2 == ')';
- case ')':
- case ')':
- return c2 == '(' || c2 == '(';
- case '>':
- case '》':
- return c2 == '<' || c2 == '《';
- case '<':
- case '《':
- return c2 == '>' || c2 == '》';
- default:
- break;
- }
- return false;
- }
-}
diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/analysis/DataAnalysisUtils.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/analysis/DataAnalysisUtils.java
deleted file mode 100644
index 86e4bbd..0000000
--- a/cl_search_api/src/main/java/com/bfd/mf/common/util/analysis/DataAnalysisUtils.java
+++ /dev/null
@@ -1,354 +0,0 @@
-package com.bfd.mf.common.util.analysis;
-
-import com.alibaba.fastjson.JSONArray;
-import com.alibaba.fastjson.JSONObject;
-import com.bfd.mf.common.util.constants.ESConstant;
-import com.bfd.mf.common.util.enums.SpreadEnums;
-import com.bfd.mf.common.web.vo.view.analysis.*;
-import com.bfd.mf.common.web.vo.view.monitor.ESMonitorEntity;
-import com.bfd.nlp.common.util.object.TObjectUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.stream.IntStream;
-
-public class DataAnalysisUtils {
-
- private static final Logger logger = LoggerFactory.getLogger(DataAnalysisUtils.class);
-
- private static final Long ONE_HOUR = 60 * 60 * 1000L;
- private static final Integer COUNT = 10;
- private static final Integer ZERO = 0;
- private static final Long ONE_DAY = 60 * 60 * 24 * 1000L;
- private static final Long EIGHT_HOUR = 60 * 60 * 8 * 1000L;
-
-
- /**
- * 根据时间,敏感等趋势图得到 pie 和 four total Count
- */
- public static JSONObject fetchPieAndCountNumberByLineJson(JSONObject jsonObject) {
-
- Long negativeNumber = 0L;
- try {
- Map> emotionsMap = (Map>) jsonObject.get("emotions");
- List negativeTimeList = emotionsMap.get(ESConstant.NEGATIVE_CN_TAG);
- for (Long number : negativeTimeList) {
- negativeNumber = negativeNumber + number;
- }
-
- PieCount emotionPieCount = fetchPieCount(emotionsMap, 0);
- Map> channelsMap = (Map>) jsonObject.get("channels");
- PieCount channelPieCount = fetchPieCount(channelsMap, 1);
-
- jsonObject.put("emotionCounts", emotionPieCount.getEmotionCounts());
- jsonObject.put("channelCounts", channelPieCount.getChannelCounts());
- jsonObject.put("negativeNumber", negativeNumber);
- } catch (Exception e) {
- logger.error("fetchPieAndCountNumberByLineJson error =", e);
- }
- return jsonObject;
- }
-
- private static PieCount fetchPieCount(Map> channelsMap, int type) {
- PieCount pieCount = new PieCount();
- if (TObjectUtils.nonNull(channelsMap)) {
- for (String name : channelsMap.keySet()) {
- long temp;
- List longs = channelsMap.get(name);
- temp = longs.stream().mapToLong(number -> number).sum();
- DataPieCount dataPieCount = new DataPieCount(name, temp);
- if (type == 0) {
- pieCount.getEmotionCounts().add(dataPieCount);
-
- } else if (type == 1) {
- pieCount.getChannelCounts().add(dataPieCount);
- }
- }
- }
- return pieCount;
- }
-
- public static Long[] getTimeList(Long startTime, Long endTime, Integer type) {
- Long[] timeList = null;
- try {
- Long interval;
- if (type.equals(SpreadEnums.TREND.UNIT_HOUR)) {
- interval = ONE_HOUR;
- } else if (type.equals(SpreadEnums.TREND.UNIT_DAY)) {
- interval = ONE_DAY;
- } else {
- throw new Exception(" !!! type is illegal !!!");
- }
- Double intervalNum = Math.ceil((endTime - startTime) / (double) interval);
- if ((endTime - startTime) % interval == 0) {
- intervalNum = intervalNum + 1;
- }
- timeList = new Long[intervalNum.intValue()];
- Long time = startTime;
- for (int i = 0; i < intervalNum; i++) {
- if (i == 0) {
- timeList[i] = time;
- } else if (i == intervalNum - 1) {
- timeList[i] = endTime;
- } else {
- time = time + interval;
- Long dayStartTime = DateTrendUtils.getDayStartTime(time);
- if (dayStartTime <= timeList[i - 1]) {
- timeList[i] = time;
- } else {
- timeList[i] = dayStartTime;
- }
- }
- }
- } catch (Exception e) {
- logger.error("[DataAnalysisUtils] getTimeList: failed, startTime is {}, endTime is {}, timeList is {}, error is ", startTime, endTime, timeList, e);
- }
- return timeList;
- }
-
- public static Map> initEmotionList(Long[] timeList) {
- Map> negativeChannelMaps = new HashMap<>();
- List emotionName = new ArrayList<>();
- emotionName.add(ESConstant.NEGATIVE_TAG);
- emotionName.add(ESConstant.COMMON_TAG);
- for (String name : emotionName) {
- for (Long aTimeList : timeList) {
- List dataLineCounts = new ArrayList<>();
- DataLineCount dataLineCount = fetchDataLineCountObject(0, name, aTimeList);
- dataLineCounts.add(dataLineCount);
- negativeChannelMaps.put(name, dataLineCounts);
- }
- }
- return negativeChannelMaps;
- }
-
- private static DataLineCount fetchDataLineCountObject(long data, String name, Long aStart) {
- DataLineCount dataLineCount = new DataLineCount();
- dataLineCount.setData(data);
- dataLineCount.setName(name);
- dataLineCount.setTimestamp(aStart);
- return dataLineCount;
- }
-
- public static void parseTimeSeries(Map> channelMaps,
- Map> emotionMaps,
- String emotionTag,
- Map> combineChannelMaps) {
- for (Map.Entry> entry : channelMaps.entrySet()) {
- List commonChannelLists = entry.getValue();
- if (Optional.ofNullable(commonChannelLists).isPresent()) {
- for (DataLineCount dataLineCount : commonChannelLists) {
- if (emotionTag.equals(ESConstant.NEGATIVE_TAG)) {
- buildEmotionResult(emotionMaps, dataLineCount, "0.0-" + ESConstant.SENTIMENTAL_THRESHOLD);
- } else if (emotionTag.equals(ESConstant.NEUTER_TAG)) {
- buildEmotionResult(emotionMaps, dataLineCount, ESConstant.SENTIMENTAL_THRESHOLD + "-" + ESConstant.SENTIMENTAL_MID);
- } else {
- buildEmotionResult(emotionMaps, dataLineCount, ESConstant.SENTIMENTAL_MID + "-" + ESConstant.SENTIMENTAL_MAX);
- }
-
- if (null != combineChannelMaps) {
- if (!combineChannelMaps.containsKey(entry.getKey())) {
- List combineChannelLists = new ArrayList<>();
- DataLineCount newDataLineCount = new DataLineCount();
- newDataLineCount.setName(dataLineCount.getName());
- newDataLineCount.setTimestamp(dataLineCount.getTimestamp());
- newDataLineCount.setData(dataLineCount.getData());
- combineChannelLists.add(newDataLineCount);
- combineChannelMaps.put(entry.getKey(), combineChannelLists);
- } else {
- List alreadyLists = combineChannelMaps.get(entry.getKey());
- if (alreadyLists.contains(dataLineCount)) {
- alreadyLists.get(alreadyLists.indexOf(dataLineCount)).setData(alreadyLists.get(alreadyLists.indexOf(dataLineCount)).getData() + dataLineCount.getData());
- } else {
- alreadyLists.add(dataLineCount);
- }
- }
- }
- }
- }
- }
- }
-
- /**
- * 计算情感
- */
- private static void buildEmotionResult(Map> emotionMaps, DataLineCount dataLineCount, String emotionTag) {
- if (!emotionMaps.containsKey(emotionTag)) {
- List negativeLists = new ArrayList<>();
- DataLineCount newDataLineCount = new DataLineCount();
- newDataLineCount.setName(dataLineCount.getName());
- newDataLineCount.setTimestamp(dataLineCount.getTimestamp());
- newDataLineCount.setData(dataLineCount.getData());
- negativeLists.add(newDataLineCount);
- emotionMaps.put(emotionTag, negativeLists);
- } else {
- List alreadyLists = emotionMaps.get(emotionTag);
-
- if (alreadyLists.contains(dataLineCount)) {
- alreadyLists.get(alreadyLists.indexOf(dataLineCount)).setData(alreadyLists.get(alreadyLists.indexOf(dataLineCount)).getData() + dataLineCount.getData());
- } else {
- DataLineCount newDataLineCount = new DataLineCount();
- newDataLineCount.setName(dataLineCount.getName());
- newDataLineCount.setTimestamp(dataLineCount.getTimestamp());
- newDataLineCount.setData(dataLineCount.getData());
- alreadyLists.add(newDataLineCount);
- }
- }
- }
-
-
- public static Map> fetchPerChannelCountByListData(List cacheEsMonitorEntityList,
- Long[] timeList, int type) {
- Map> channelMaps = new HashMap<>();
- for (ESMonitorEntity esMonitorEntity : cacheEsMonitorEntityList) {
- String name = esMonitorEntity.getDocType();
- DataAnalysisUtils.fetchFilterResultByTimeCompare(channelMaps, name, esMonitorEntity, timeList, type);
- }
- return channelMaps;
- }
-
- /**
- * 时间段统计数量
- */
- public static void fetchFilterResultByTimeCompare(Map> negativeChannelMaps,
- String name,
- ESMonitorEntity esMonitorEntity,
- Long[] timeList,
- int type) {
- Long pubTime = esMonitorEntity.getPubTime();
- int length = timeList.length;
- boolean isZeroTime = false;
- for (int i = 0; i < length; i++) {
- Long aStart;
- Long aEnd = null;
- if (type == 1) {
- // 针对天
- if (i == length - 1) {
- long temp;
- aStart = timeList[i];
- isZeroTime = isZeroTime(aStart);
- if (isZeroTime) {
- aEnd = timeList[i];
- } else {
- temp = getDayTime(aStart);
- if (aStart < temp) {
- // 大于8小时
- if (temp > aStart) {
- temp = temp - ONE_DAY;
- }
- aEnd = aStart;
- aStart = temp;
- } else {
- aEnd = aStart;
- aStart = temp;
- }
- }
-
- } else if (i == length - 2) {
- long temp;
- aStart = timeList[i];
- temp = getDayTime(aStart);
- if (aStart <= temp) {
- aEnd = aStart + ONE_DAY;
- }
- } else {
- aStart = timeList[i];
- aEnd = timeList[i + 1];
- }
- } else {
- if (i == length - 1) {
- aStart = timeList[i];
- aEnd = aStart + ONE_DAY;
- } else {
- aStart = timeList[i];
- aEnd = timeList[i + 1];
- }
- }
-
- if (pubTime >= aStart && (pubTime < aEnd || isZeroTime)) {
- long temp = aStart;
- // 天,零点,
- if (i == length - 1 && !isZeroTime && type == 1) {
- temp = aEnd;
- }
- if (negativeChannelMaps.containsKey(name)) {
- List dataLineCounts = negativeChannelMaps.get(name);
- boolean existTime = false;
- for (DataLineCount dataLineCount : dataLineCounts) {
- Long timestamp = dataLineCount.getTimestamp();
- if (Objects.equals(timestamp, temp)) {
- dataLineCount.setData(dataLineCount.getData() + 1);
- existTime = true;
- break;
- }
- }
- if (!existTime) {
- DataLineCount dataLineCount = fetchDataLineCountObject(1L, name, temp);
- dataLineCounts.add(dataLineCount);
- negativeChannelMaps.put(name, dataLineCounts);
- }
- } else {
- List dataLineCounts = new ArrayList<>();
- DataLineCount dataLineCount = fetchDataLineCountObject(1L, name, temp);
- dataLineCounts.add(dataLineCount);
- negativeChannelMaps.put(name, dataLineCounts);
- }
- break;
- }
- }
- }
- private static boolean isZeroTime(Long time) {
- SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss");
- return dateFormat.format(time).equals("00:00:00");
-
- }
- private static long getDayTime(Long aStart) {
- long dayTime = aStart - (aStart) % ONE_DAY;
- dayTime = dayTime - EIGHT_HOUR;
- dayTime = dayTime + ONE_DAY;
- return dayTime;
- }
-
-
- public static Long[] fetchPerTimeByPerChannelCountResult(Map> channelPerTimeMap, Long[] timeArray) {
- Long[] docNumList = IntStream.range(0, timeArray.length).mapToObj(i -> 0L).toArray(Long[]::new);
- for (int i = 0; i < timeArray.length; i++) {
- long temp = 0;
- for (String key : channelPerTimeMap.keySet()) {
- temp = channelPerTimeMap.get(key).get(i) + temp;
- }
- docNumList[i] = temp;
- }
- return docNumList;
- }
-
- public static JSONArray getEmotionOrChannelTrendTime(List dataList,
- Long[] timeList,
- JSONArray timeArr,
- int type) throws ParseException {
- if (TObjectUtils.isListEmpty(dataList)) {
- return null;
- }
- Long[] docNumList = new Long[dataList.size()];
- for (int i = 0; i < dataList.size(); i++) {
- docNumList[i] = dataList.get(i);
- }
- JSONArray countByHourData = null;
- switch (type) {
- case 1:
- countByHourData = DateTrendUtils.getDayCountByHourData(timeList, timeArr, docNumList);
- break;
- case 2:
- countByHourData = DateTrendUtils.getMonthCountByHourData(timeList, timeArr, docNumList);
- break;
- case 3:
- countByHourData = DateTrendUtils.getYearCountByHourData(timeList, timeArr, docNumList);
- default:
- break;
- }
- return countByHourData;
- }
-}
\ No newline at end of file
diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/analysis/DateTrendUtils.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/analysis/DateTrendUtils.java
deleted file mode 100644
index 9046502..0000000
--- a/cl_search_api/src/main/java/com/bfd/mf/common/util/analysis/DateTrendUtils.java
+++ /dev/null
@@ -1,279 +0,0 @@
-package com.bfd.mf.common.util.analysis;
-
-import com.alibaba.fastjson.JSONArray;
-
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Calendar;
-import java.util.GregorianCalendar;
-import java.util.LinkedHashSet;
-import java.util.Set;
-
-public class DateTrendUtils {
-
-
- /**
- * 获得该月第一天
- */
- public static String getFirstDayOfMonth(Integer year, Integer month) throws ParseException {
- Calendar cal = Calendar.getInstance();
- //设置年份
- cal.set(Calendar.YEAR, year);
- //设置月份
- cal.set(Calendar.MONTH, month - 1);
- //获取某月最小天数
- int firstDay = cal.getActualMinimum(Calendar.DAY_OF_MONTH);
- //设置日历中月份的最小天数
- cal.set(Calendar.DAY_OF_MONTH, firstDay);
- //格式化日期
- SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
- return sdf.format(cal.getTime());
- }
-
- /**
- * 获得该月最后一天
- */
- private static String getLastDayOfMonth(Integer year, Integer month) throws ParseException {
- Calendar cal = Calendar.getInstance();
- //设置年份
- cal.set(Calendar.YEAR, year);
- //设置月份
- cal.set(Calendar.MONTH, month - 1);
- //获取某月最大天数
- int lastDay = cal.getActualMaximum(Calendar.DAY_OF_MONTH);
- //设置日历中月份的最大天数
- cal.set(Calendar.DAY_OF_MONTH, lastDay);
- //格式化日期
- SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
- return sdf.format(cal.getTime());
- }
-
-
- /**
- * 得到月的统计值
- */
- public static JSONArray getMonthCountByHourData(Long[] timeList, JSONArray monthArr, Long[] countArr) throws ParseException {
- SimpleDateFormat format = new SimpleDateFormat("yyyy-MM");
- // 存储 Mark
- JSONArray markArr = new JSONArray();
- for (int i = 0; i < monthArr.size(); i++) {
- Long dayEndTime = getMonthEndDay(getDayEndTime(format.parse(monthArr.getString(i)).getTime()));
- int mark = 0;
- for (int j = 0; j < timeList.length; j++) {
- if (dayEndTime < timeList[j]) {
- mark = j;
- break;
- } else if (dayEndTime.equals(timeList[j])) {
- mark = j + 1;
- break;
- }
- }
- mark = mark == 0 ? timeList.length : mark;
- markArr.add(mark);
- }
- // new data
- return getNewCountData(markArr, countArr);
- }
-
- /**
- * 得到天的统计值
- */
- public static JSONArray getDayCountByHourData(Long[] timeList, JSONArray dayArr, Long[] countArr) throws ParseException {
- SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
- // 存储 Mark
- JSONArray markArr = new JSONArray();
- for (int i = 0; i < dayArr.size(); i++) {
- Long dayEndTime = getDayEndTime(format.parse(dayArr.getString(i)).getTime());
- int mark = 0;
- for (int j = 0; j < timeList.length; j++) {
- if (dayEndTime < timeList[j]) {
- mark = j;
- break;
- } else if (dayEndTime.equals(timeList[j])) {
- mark = j + 1;
- break;
- }
- }
- mark = mark == 0 ? timeList.length : mark;
- markArr.add(mark);
- }
-
- // new data
- return getNewCountData(markArr, countArr);
- }
-
- /**
- * 得到年的统计值
- */
- public static JSONArray getYearCountByHourData(Long[] timeList, JSONArray yearArr, Long[] countArr) throws ParseException {
- SimpleDateFormat format = new SimpleDateFormat("yyyy");
- // 存储 Mark
- JSONArray markArr = new JSONArray();
- for (int i = 0; i < yearArr.size(); i++) {
- Long dayEndTime = getDayEndTime(getYearEndDay(format.parse(yearArr.getString(i)).getTime()));
- int mark = 0;
- for (int j = 0; j < timeList.length; j++) {
- if (dayEndTime < timeList[j]) {
- mark = j;
- break;
- } else if (dayEndTime.equals(timeList[j])) {
- mark = j + 1;
- break;
- }
- }
- mark = mark == 0 ? timeList.length : mark;
- markArr.add(mark);
- }
- // new data
- return getNewCountData(markArr, countArr);
- }
-
- /**
- * 计算得到的统计值
- */
- private static JSONArray getNewCountData(JSONArray markArr, Long[] countArr) {
- JSONArray newData = new JSONArray();
- for (int j = 0; j < markArr.size(); j++) {
- int startIndex = 0;
- int endIndex;
- if (j == 0) {
- endIndex = markArr.getInteger(j);
- } else {
- endIndex = markArr.getInteger(j);
- startIndex = markArr.getInteger(j - 1);
- }
-
- long count = 0;
- for (; startIndex < endIndex; startIndex++) {
- count = count + countArr[startIndex];
- }
- newData.add(count);
- }
- return newData;
- }
-
- /**
- * 一天的开始时间
- */
- public static Long getDayStartTime(Long time) {
- Calendar todayStart = new GregorianCalendar();
- todayStart.setTimeInMillis(time);
-
- todayStart.set(Calendar.HOUR_OF_DAY, 0);
- todayStart.set(Calendar.MINUTE, 0);
- todayStart.set(Calendar.SECOND, 0);
- todayStart.set(Calendar.MILLISECOND, 0);
- return todayStart.getTime().getTime();
-
- }
-
- /**
- * 一天的结束时间
- */
- public static Long getDayEndTime(Long time) {
- Calendar todayEnd = new GregorianCalendar();
- todayEnd.setTimeInMillis(time);
-
- todayEnd.set(Calendar.HOUR_OF_DAY, 23);
- todayEnd.set(Calendar.MINUTE, 59);
- todayEnd.set(Calendar.SECOND, 59);
- todayEnd.set(Calendar.MILLISECOND, 999);
- return todayEnd.getTime().getTime();
-
- }
-
- /**
- * 月的最后一天
- */
- private static Long getMonthEndDay(Long time) {
- Calendar ca = new GregorianCalendar();
- ca.setTimeInMillis(time);
- ca.set(Calendar.DAY_OF_MONTH, ca.getActualMaximum(Calendar.DAY_OF_MONTH));
- return ca.getTime().getTime();
- }
-
- /**
- * 月的第一天
- */
- public static Long getMonthStartDay(Long time) {
- Calendar c = new GregorianCalendar();
- c.setTimeInMillis(time);
- c.add(Calendar.MONTH, 0);
- c.set(Calendar.DAY_OF_MONTH, 1); //设置为1号,当前日期既为本月第一天
- return c.getTime().getTime();
- }
-
- /**
- * 年的最后一天
- */
- private static Long getYearEndDay(Long time) {
- Calendar c = new GregorianCalendar();
- c.setTimeInMillis(time);
- c.set(Calendar.DAY_OF_YEAR, c.getActualMaximum(Calendar.DAY_OF_YEAR));
- return c.getTime().getTime();
-
- }
-
- /**
- * 年的第一天
- */
- public static Long getYearStartDay(Long time) {
- Calendar c = new GregorianCalendar();
- c.setTimeInMillis(time);
- c.add(Calendar.YEAR, 0);
- c.set(Calendar.DAY_OF_YEAR, 1);
-
- return c.getTime().getTime();
-
- }
-
-
- /**
- * 得到天,月,年数组
- */
- private static JSONArray getStrTime(Long[] timeList, SimpleDateFormat format) {
- Set setTime = new LinkedHashSet<>();
- for (Long aLong : timeList) {
- String s = format.format(aLong);
- setTime.add(s);
- }
- JSONArray arrayTime = new JSONArray();
- arrayTime.addAll(setTime);
- return arrayTime;
- }
-
- /**
- * 转换 day,month,year arr
- */
- public static JSONArray convertStrTime(Long[] timeList, int type) {
- JSONArray arrayTime;
- SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
- switch (type) {
- case 1:
- // day
- format = new SimpleDateFormat("yyyy-MM-dd");
- break;
- case 2:
- // month
- format = new SimpleDateFormat("yyyy-MM");
- break;
- case 3:
- // year
- format = new SimpleDateFormat("yyyy");
- break;
- default:
- break;
- }
- arrayTime = getStrTime(timeList, format);
- return arrayTime;
- }
-
- /**
- * 获取小时
- */
- public static String getHourByLongTime(Long longTime) {
- SimpleDateFormat format = new SimpleDateFormat("HH");
- return format.format(longTime);
- }
-
-}
diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/cache/ResponseParseUtils.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/cache/ResponseParseUtils.java
deleted file mode 100644
index 76bb676..0000000
--- a/cl_search_api/src/main/java/com/bfd/mf/common/util/cache/ResponseParseUtils.java
+++ /dev/null
@@ -1,43 +0,0 @@
-package com.bfd.mf.common.util.cache;
-
-import com.bfd.mf.common.util.es.MonitorUtils;
-import com.bfd.mf.common.web.vo.view.monitor.ESMonitorBaseEntity;
-import com.bfd.nlp.common.util.object.TObjectUtils;
-import org.elasticsearch.action.search.SearchResponse;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-public class ResponseParseUtils {
- private static Logger logger = LoggerFactory.getLogger(ResponseParseUtils.class);
-
- public static long combineEsAndCatchResponse(
- List cacheResponse,
- Map timeSeriesMap) {
- Map idSet = combineEsAndCatchResponseWithDocId(
- cacheResponse,
- timeSeriesMap);
- return idSet.size();
- }
-
- public static Map combineEsAndCatchResponseWithDocId(
- List cacheResponse,
- Map timeSeriesMap) {
- Map docIdWithAreaMap = new HashMap<>();
- if (!TObjectUtils.isListEmpty(cacheResponse)) {
- cacheResponse.forEach(response ->
- MonitorUtils.parseTimeSeries(
- response,
- docIdWithAreaMap,
- timeSeriesMap)
- );
- }
- logger.info("[combineEsAndCatchResponseWithDocIdCrawl method] the all doc number is {},the folder doc number is {}", docIdWithAreaMap.size(), timeSeriesMap.size());
- return docIdWithAreaMap;
- }
-
-}
diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ConditionCommon.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ConditionCommon.java
deleted file mode 100644
index 662e641..0000000
--- a/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ConditionCommon.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package com.bfd.mf.common.util.constants;
-
-public class ConditionCommon {
- public static final String SPLIT = "_";
-
- public static final Integer HOUR = 0;
- public static final Integer DAY = 1;
- public static final Integer MONTH = 2;
- public static final Integer YEAR = 3;
-
- public static final String PROVINCE = "province";
- public static final String CITY = "city";
-
-
- // other
- public static final String MEDIA_COUNTS = "mediaCounts";
- public static final String MEDIA_COUNTS_ALL = "mediaCountsALL";
- public static final String HOUR_COUNT_MAP = "hourCountMap";
- public static final String WORD_CLOUD = "wordCloud";
- public static final String PLACE_CLOUD = "placesCloud";
- public static final String HASH_TAG_CLOUD = "hashTagCloud";
- public static final String EMOJI_CLOUD = "emojiCloud";
- public static final String OPINION_CLOUD = "opinionsCloud";
-
-
- // weibo
- public static final String VERIFIED_COUNTS = "verifiedCounts";
- public static final String SEX_COUNTS = "sexCounts";
- public static final String AGE_COUNT_MAP = "ageCountMap";
- public static final String WEIBO_COUNT = "weiboCount";
-
- public static final String NORMAL = "常规";
- public static final String PROMOTION = "推广";
-
-
-}
diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ESConstant.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ESConstant.java
deleted file mode 100644
index a04334d..0000000
--- a/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ESConstant.java
+++ /dev/null
@@ -1,663 +0,0 @@
-/*
- * Copyright (C) 2016 Baifendian Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.bfd.mf.common.util.constants;
-
-import com.bfd.mf.common.util.utility.EsQueryConditionUtils;
-import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class ESConstant {
-
-
- public static final String MEDIA_AREA_KEY = "dict";
-
- public static final float BOOST_WEIGHT = 10f;
- // 相似查询返回的最大数据量
- public static final Integer MAX_RESULT_WINDOWS_SIZE = 10000;
- public static final Integer MAX_SCROLL_SIZE = 5000;
- public static final Integer MAX_SCORLL_SIZE_CL = 2000;
- public static final Long MAX_SEARCH_TIME_MINUTES = 2L;
- public static final String SUBJECT_CLUSTER_INDEX_PREFIX = "mf_subject";
- public static final String ES_KEY = "es";
- public static final String ES_NORMAL_KEY = "es_normal";
- public static final String CACHE_KEY = "cache";
- public static final int BULK_SIZE = 100;
- public static final String INDEX_TYPE = "docs";
- public static final String bigramPostfix = ".shingles";
- public static final String bigramKey = "bigram";
- public static final String filterTypeKey = "filter_type";
- public static final String queryTypeKey = "query_type";
- public static final Double aggOffset = 16 * 6
- * 60 * 1000d;
- public static final Double ONE_DAY = 24 * 60 * 60 * 1000d;
- public static final Double ONE_HOUR = 60 * 60 * 1000d;
-
- public static final String SENTIMENT = "sentiment";
-
- public static final String SUMMARY = "summary";
-
-
- public static final String FILEPATH = "filePath";
- public static final String ISDOWNLOAD = "isDownload";
- public static final String VIDEOURL = "videoUrl";
-
- /**
- * *
- *
- * @FIXED 2017.11.13 increase the runtime,from 3 min to 6 min
- */
- public static final long THREAD_TIME = 15 * 60 * 1000L;
-
- public static String DESC = "desc";
-
- public static String ASC = "asc";
-
- /**
- * pubTime时间由远及近
- */
- public static String DEFUALT_SORT = "createdTimeDesc";
-
- /**
- * pubTime时间由远及近
- */
- public static String PUBTIME_ASC_SORT_FLAG = "timeAsc";
-
-/**
- * 评论数:commentDescFlag
- 发帖时间:(近-远)timeDesc
- 发帖时间:(远-近) timeAsc
- 采集时间:crawlTimeDesc
- */
- /**
- * pubTime时间由近及远
- */
- public static String PUBTIME_DESC_SORT_FLAG = "timeDesc";
- public static String CRAWLTIME_DESC_SORT_FLAG = "crawlTimeDesc";
-
- /**
- * createTime时间由近及远
- */
- public static String CREATE_TIME_DESC_SORT_FLAG = "createdTimeDesc";
-
- /**
- * createTime时间由远及近
- */
- public static String CREATE_TIME_ASC_SORT_FLAG = "createdTimeAsc";
-
- /**
- * 重要度排序
- */
- public static String SCORE_DESC_SORT_FLAG = "scoreDesc";
-
- /**
- * Alexa
- */
- public static String ALEXA_ASC_SORT_FLAG = "alexaAsc";
- /***
- *
- */
- public static Float ALEXA_DEFAULT_SCORE = 1000000.00F;
- /**
- * 转载量排序
- */
- public static String FOLDER_NUMBER_DESC_SORT_FLAG = "folderNumberDescSortFlag";
- /**
- * 综合排序:1:转载量排序;2:转载量一样按照得分排序;3:随后按照pubTime时间倒序
- */
- public static String REPORT_SORT_FLAG = "reportSortFlag";
-
- public static String SCORE = "_score";
- public static String SCORE_FROM_SOURCE = "score";
-
-
- /**
- * 关键词前缀
- */
- public static final String TAG_START = "";
- /**
- * 关键词后缀
- */
- public static final String TAG_END = "";
-
- /**
- * 标题关键词分隔符
- */
- public static final String TITLE_KEYWORDS_SPLIT_SYSBOL = ",";
- /**
- * 相关度等级词分隔符
- */
- public static final String RELATEDWORDS_HIGH = "high";
- public static final String RELATEDWORDS_MEDIUM = "medium";
- public static final String RELATEDWORDS_LOW = "low";
- /**
- * 相关度分级
- */
- public static final float RELATEDRATE_MIN = 0.00F;
- public static final float RELATEDRATE_MAX = 1.00F;
- public static final int RELATEDRATE_MIN_SEARCH_NUMBER = 10;
- /**
- * 排除关键词分隔符
- */
- public static final String EXCLUDE_KEYWORDS_SPLIT_SYSBOL = ",";
- public static final String EXCLUDE_KEYWORDS_SPLIT_CN_SYSBOL = ",";
-
-
- /**
- * 最小匹配度
- */
- public static final Integer MINI_NUMBER_SHOULD_MATCH = 1;
- /**
- * 最大摘要长度
- */
- public static final Integer MAX_FRAGMENT_SIZE = 10000;
- /**
- * 数据监控页面摘要长度
- */
- public static final Integer MONITOR_FRAGMENT_SIZE = 110;
- // 报告摘要长度
- public static final Integer Report_FRAGMENT_SIZE = 100;
- public static final Integer Foreign_Report_FRAGMENT_SIZE = 200;
- /**
- * 数据监控页面标题高亮长度
- */
- public static final Integer MONITOR_FRAGMENT_TITLE_SIZE = 50;
- /**
- * 数据导出查询大集群分页查询,每次最大查询量
- */
- public static final Integer EXPORT_SCROLL_SIZE = 1000;
- /**
- * 自定义长度
- */
- // title 长度
- public static final Integer MONITOR_FRAGMENT_USER_DEFINED_TITLE_SIZE = 30;
- // 系统报告自定义长度 40
- public static final Integer SYS_REPORT_FRAGMENT_USER_DEFINED_TITLE_SIZE = 40;
- public static final Integer FOREIGN_SYS_REPORT_FRAGMENT_USER_DEFINED_TITLE_SIZE = 80;
-
- public static final Integer MAX_FRAGMENT_USER_DEFINED_TITLE_SIZE = 50;
-
- // 预警报告的摘要长度
- public static final Integer MONITOR_FRAGMENT_USER_DEFINED_DIGEST_SIZE = 150;
- public static final String DEFAULT_JOIN_SYMBOL = ".";
- /**
- * 主体情感默认名称
- */
- public static final String DEFAULT_ENTRY_NAME = "default";
- public static final String EMOTION_ENTRY = "emotionEntry";
- public static final String EMOTION_VALUE = "emotionValue";
- public static final String EMOTION_NAME = "entryName";
- public static final String OFF_SET = "offSet";
- public static final String MINIMUM_SHOULD_MATCH = "1";
- /*
- * docType(news\bbs...):int 类型常量
- */
- public static final int DOCTYPENEWS = 0;
- public static final int DOCTYPEBBS = 1;
- /*
- * 组合查询语句key值类型
- */
- public static final String KEYTYPEOPERA = "operator";
- public static final String KEYTYPEWORD = "word";
- /*
- * 数据来源渠道 (hl \ bfd)
- */
- public static final String CHANNELHL = "HL";
- public static final String CHANNELBFD = "BFD";
- /*
- *1:type:content、title、all
- *2: text:words(目前要是关键词列表)
- */
- public static final String SEARCH_SCOPE_TYPE_TITLE = "title";
- public static final String ICON = "icon";
-
- public static final String SEARCH_SCOPE_TYPE_CONTENT = "content";
-
- //标题+正文
- public static final String SEARCH_SCOPE_TYPE_TITLE_ALL = "title_all";
-
- public static final String SEARCH_SCOPE_TYPE_AUTHOR = "author";
-
- public static final String SEARCH_SCOPE_TYPE_FORWARD = "forwardContent";
-
- public static final String SEARCH_SCOPE_TYPE_TITLE_ORIGIN = "title_origin";
-
- public static final String SEARCH_SCOPE_TYPE_CONTENT_ORIGIN = "content_origin";
-
- public static final String SEARCH_SCOPE_TYPE_ALL = "all";
-
- public static final String SEARCH_SCOPE_WECHAT_SUBSCRIPTION = "wechat_subscription";
-
- /**
- * SQ 4.0新增 UGC EGC PGC
- */
- public static final String SEARCH_SCOPE_UGC = "ugc";
-
- public static final String SEARCH_SCOPE_EGC = "egc";
-
- public static final String SEARCH_SCOPE_PGC = "pgc";
- /**
- * 关键词
- */
- public static final String SEARCH_SCOPE_TYPE_KEYWORDS = "keyWords";
-
- public static final String REL_TYPE = "relType";
-
- public static final String PRIMARY_POST = "primaryPost";
-
- public static final String PRIMARY = "primary";
-
- public static final String PT_PRIMARY = "ptPrimary";
-
- public static final String DT_PRIMARY = "dtPrimary";
-
- public static final String PT_ALL = "ptAll.keyword";
-
- public static final String DT_ALL = "dtAll.keyword";
-
- public static final String DATA_COUNT = "dataCount";
-
- /**
- * 回溯开始时间
- */
- public static final String CON_START_TIME = "conStartTime";
- /**
- * 回溯结束时间
- */
- public static final String CON_END_TIME = "conEndTime";
- public static final String SUBJECT_ID = "subjectId";
- /*
- *1:非敏感 positive
- *2:敏感 negative
- */
- public static final String COMMON_TAG = "common";
- public static final String NEGATIVE_TAG = "negative";
- public static final String NEUTER_TAG = "neuter";
-
- public static final Integer COMMON_TAG_VALUE = 1;
- public static final Integer NEGATIVE_TAG_VALUE = 3;
- public static final Integer NEUTER_TAG_VALUE = 2;
-
- public static final String COMMON_CN_TAG = "正面";
- public static final String NEGATIVE_CN_TAG = "负面";
- public static final String NEUTER_CN_TAG = "中性";
-
- public static final String AVAILABILITY = "availability";
-
- public static final List SENTIMENT_LIST = new ArrayList();
- public static String URL_HASH = "urlHash";
-
- static {
- SENTIMENT_LIST.add(COMMON_TAG);
- SENTIMENT_LIST.add(NEGATIVE_TAG);
- SENTIMENT_LIST.add(NEUTER_TAG);
- }
-
- // 正面 [0.8-1]
- // 中性(0.2-0.8)
- // 负面 [0-0.2]
- public static final Double SENTIMENTAL_MID = 0.8;
- public static final Double SENTIMENTAL_THRESHOLD = 0.2;
- public static final Double SENTIMENTAL_MIN = 0.0;
- public static final Double SENTIMENTAL_MAX = 1.0;
- public static final String OPERATOR_OR = "or";
- public static final String OPERATOR_EQUAL = "=";
- public static final String OPERATOR_SPACE = " ";
- public static final String OPERATOR_ALL = "*:*";
- public static final String OPERATOR_CONNENT_ALL = ":";
- public static final String OPERATOR_SINGLE_ALL = "*";
-
-
- /*
- *define es mapping fields
- */
- public static String PUBTIME = "pubTime";
- public static String CHANNEL_HYLANDA = "hylanda";
-
- public static String PUBTIME_STR = "pubTimeStr";
-
- public static String CREATETIME = "createTime";
- public static String CREATETIMESTR = "createTimeStr";
- public static String CREATETIMEDAY = "createDay";
-
- public static String PUTTIME = "putTime";
- public static String CRAWLTIME = "crawlTime";
- public static String ORIGINAL_SOURCE = "originalSource";
- public static String CONTENT_SIMHASH = "contentSimHash";
- public static String QUOTE_COUNT = "quoteCount";
- /**
- * 内容
- */
- public static String CONTENT_TAG = "contentTag";
- public static String DT_EN_2 = "dtEn2";
- public static String DT_EN_1 = "dtEn1";
- public static String COMMENTS_COUNT = "commentsCount";
- public static String PAGE_COMMENTS_COUNT = "pageCommentCount";
- public static String ATTITUDES_COUNT = "attitudesCount";
- public static String TITLE_SIMHASH = "titleSimHash";
- public static String ORIGINAL_CONTENT_SIMHASH = "originalContentSimHash";
- public static String ORIGINAL_TITLE_SIMHASH = "originalTitleSimHash";
- public static String DOC_TYPE = "docType";
- public static String CHINA_DOC_TYPE = "chinaDocType";
- public static String SYS_SENTIMENT_TAG = "sysSentimentTag";
- public static final String TITLE = "title";
- public static final String USER_LEVEL_NAME = "userLevelName";
- public static String SOURCE = "source";
- public static String CRAWLDATAFLAG= "crawlDataFlag";
-
- //微信专属字段,内容固定 value="微信"
- public static String WEIXIN_SOURCE = "weixinSource";
- public static String CHANNEL = "channel";
- public static final String CONTENT = "content";
- public static final String SYS_SENTIMENT = "sysSentiment";
- public static String POST_SOURCE = "postSource";
- /**
- * DT情感标签
- */
- public static final String DT_SENTIMENT = "dtSentiment";
- /**
- * PT情感标签
- */
- public static final String PT_SENTIMENT = "ptSentiment";
-
- public static String URL = "url";
- public static String DOC_ID = "docId";
- public static String DATA_ID = "dataId";
- public static String _ID = "_id_";
- public static String SYS_ABSTRACT = "sysAbstract";
- public static String SYS_KEYWORDS = "sysKeywords";
- // 所有热词
- public static String HL_KEYWORDS = "hlKeywords";
- public static String ALL_KEYWORDS = "allKeywords";
- public static String COMMENT_KEY_WORDS = "commentKeywords";
- public static String OPTIONS = "opinions";
- // 地名:places
- public static String PLACES = "places";
- // 微博话题:hashTag
- public static String HASH_TAG = "hashTag";
- public static String PICTURE_LIST = "pictureList";
-
- // 表情:expression
- public static String EXPRESSION = "expression";
- // 评价:opinions
- public static String OPINIONS = "opinions";
-
- public static String WEIBO_AUTHOR = "author";
- public static String WEIBO_USER_ID = "userId";
- public static String AUTHOR = "author";
- public static String WEIXIN_NAME = "weixinName";
- public static final String CHANNEL_WEIXIN = "weixin";
- public static final String HL_DOCID = "hylandaDocId";
- public static final String LANGUAGE = "language";
- public static String AREA = "area";
-
-
- /**
- * _all字段
- */
- public static String _ALL = "_all";
-
- /**
- * 口碑
- */
- public static final String SHOP_MD5 = "shopUrlHash";
- public static final String HAS_PICTURE = "isCommentImg";
- public static final String HAS_CONTENT = "isComment";
- public static final String CUSTOMER_LEVEL = "customerLevel";
- public static final String COMMENT_STAR_LEVEL = "starLevel";
- public static final String SHOP_NAME = "shopName";
- public static final String NICKNAME = "nickName";
- public static final String COMMENT_SEARCH_ALL = "comment_all";
- public static final String USER_AVATAR = "userAvatar";
- public static final String UNIQUE_ID = "uniqueId";
- public static final String SHOP_URL = "shopUrl";
- public static final String USER_URL = "userUrl";
- public static final String CONTRIBUTION = "contribution";
- public static final String SCORE_FACTOR = "scoreFactor";
- public static final String FACTOR_KEY = "factorKey";
- public static final String FACTOR_VALUE = "factorValue";
- public static final String CONTENT_SIZE = "commentLength";
- public static final String SHOP_LABEL = "label";
- public static final String SHOP_CITY = "shopCity";
- public static final String SHOP_PROVINCE = "shopProvince";
- public static final String USER_CITY = "userCity";
- public static final String COUNTRY = "country";
- public static final String USER_PROVINCE = "userProvince";
- public static final String SHOP_LABELS = "shopLabels";
- public static final String COMMENT_IMG = "commentImg";
-
- /**
- * 用户头像
- */
- public static String WEIBO_AVATOR = "avatar";
- public static final String SOCIAL = "social";
- public static final String BBS = "bbs";//论坛
- public static final String WEI_BO = "weibo";//微博
- public static final String WEI_XIN = "weixin";//微信
- public static final String CH_WEI_XIN = "微信";
- public static final String BLOG = "blog";//博客
- public static final String NEWS = "news";//网站
- public static final String TIEBA = "tieba";//贴吧
- public static final String MEDIA = "media";//报刊
- public static final String VIDEO = "video";//视频
- public static final String WENDA = "wenda";//问答
- public static final String SHEJIAO = "shejiao";// 社交
- public static final String APP = "app";
- public static String WEIBO_DEFAULT_AVATOR = "http://tva1.sinaimg.cn/default/images/default_avatar_male_180.gif";
-
- public static final String BAIDUKOUBEI = "baidukoubei";
- public static final String DIANPING = "dianping";
- public static final String FANGTIANXIA = "fangtianxia";
- public static final String BAIDUTIEBA = "baidutieba";
- public static final String SOUHU = "souhu";
- public static final String XIECHENG = "xiecheng";
- public static final String AUTOHOME = "autohome";
- public static final String TMALL = "tmall";
- public static final String YILONG = "yilong";
- // 电商
- public static final String ITEM = "item";
- public static final String ITEM_NAME = "itemName";
- public static final String ZIXUN = "zixun";
- public static final String FOREIGN_NEWS = "foreign_news";
- public static final String FOREIGN_MEDIA = "foreign_media";
- //电商星级分数
- public static final String COMMENT_SCORE = "commentScore";
- // 电商数据没有星级时返回-1
- public static final Double COMMENT_SCORE_NULL = -1.0;
- public static final String EN_SOURCE = "enSource";
-
- //内容识别
- public static final String NORMAL = "normal";
- public static final String MARKETING = "marketing";
-
- public static final String AVATAR = "avatar";
- public static final String COMMENT_ID = "commentId";
-
-
- /**
- * 微博信息 added by Eric 2016-10-28 15:03:31
- */
- public static String WEIBO_USER_SEX = "sex";// 性别 female \ male
- public static String WEIBO_USER_SEX_FEMALE = "female";// 性别 female
- public static String WEIBO_USER_SEX_MALE = "male";
-
- public static String WEIBO_USER_FRIENDS_COUNT = "friendsCount";// 关注数
- public static String WEIBO_USER_FANS_COUNT = "fansCount";// 粉丝数
- public static String WEIBO_ATTITUDES_COUNT = "attitudesCount";// 赞数
- public static String WEIBO_COMMENTS_COUNT = "commentsCount";// 评论数
- public static String WEIBO_QUOTE_COUNT = "quoteCount";// 转发数
- public static String WEIBO_COUNT = "weiboCount";//用户转发微博数量
- public static String WEIBO_LOCATION = "location";//用户转发微博数量
- public static String WEIBO_USER_COUNTRY = "country";// 国家
- public static String WEIBO_USER_PROVINCE = "province";// 省份
- public static String WEIBO_USER_CITY = "city";// 城市 verified
- public static String WEIBO_USER_AREA = "area";
- public static String WEIBO_MUNICIPALITY_BEIJING = "北京";
- public static String WEIBO_MUNICIPALITY_TIANJIN = "天津";
- public static String WEIBO_MUNICIPALITY_CHONGQING = "重庆";
- public static String WEIBO_MUNICIPALITY_SHANGHAI = "上海";
-
- public static String WEIBO_USER_VERIFIED = "verified";// 认证类型
- public static String WEIBO_USER_NOT_VERIFIED_VALUE = "0";// 非认证类型
- public static String WEIBO_ORIGINAL_URL = "originalUrl";// 上层url
- public static String WEIBO_EXPRESSION = "expression";// 表情图
- public static String WEIBO_EXPRESSION_TEXT = "expressionText";// 表情文本
- public static String WEIBO_FORWARD_URL = "forwardUrl";// 原文url
- public static String FORWARD_USER_URL = "forwardUserUrl";// 原文url
- public static String WEIBO_USER_TYPE = "userType";
- public static String WEIBO_POST_SOURCE = "postSource";
- public static String WEIBO_LEVEL = "level";
- public static String WEIBO_REPLY_COMMENT = "replycomment";
- public static String PRODUCT_ID = "productId";
-
-
- /**
- * 微博转发信息
- */
- public static final String FORWARD_CONTENT = "forwardContent"; // 转发原文内容
- public static final String FORWARD_RELEASE_DATE = "forwardReleaseDate"; // 转发原文发布时间
- public static final String FORWARD_AUTHOR = "forwardAuthor"; // 转发原文作者昵称
-
- public static final String FORWARD_QUOTE_COUNT = "forwardQuoteCount";
- public static final String FORWARD_POST_SOURCE = "forwardPostSource";
- public static final String FORWARD_COMMENT_COUNT = "forwardCommentsCount";
- public static final String FORWARD_ATTITUDES_COUNT = "forwardAttitudesCount";
- public static final String FORWARD_URL = "forwardUrl";
- public static final String FORWARD_AVATAR = "forwardAvatar";
- public static final String FORWARD_USER_TYPE = "forwardUserType";
- public static final String FORWARD_PUBTIME = "forwardPubTime";
-
-
- public static final String FORWARD_FLAG = "forwardFlag"; // 转发、回帖标志 1(转发、回帖) 0(原贴)
-
- public static final int REGULAR_PUBTIME_QUERY = 0;
-
- public static final int REGULAR_PUBTIME_AND_CREATETIME_QUERY = 1;
-
-
- public static final List TYPE_LIST = new ArrayList<>();
-
- public static final List BACKEND_ANALYSIS_LIST = new ArrayList<>();
-
- public static final List BACKEND_AVAILABILITY_LIST = new ArrayList<>();
-
-
- public static final List APP_TYPE_LIST = new ArrayList();
-
-
- /**
- * 口碑评价对应渠道类型
- */
- public static final List ITEM_TYPE_LIST = new ArrayList();
-
- private static final String MEITUAN = "meituan";
-
- private static final String DIAPING = "dianping";
-
- public static final String ID = "id";
-
- public static final String COMMENTS = "comments";
-
- public static final String REPOSTS = "reposts";
-
- public static final String ATTITUDES = "attitudes";
-
- public static final Map SORT_FLAG_MAPPING_MYSQL_SORT = new HashMap<>();
-
- public static final String[] FIELD_LIST_MONITOR_NORMAL = EsQueryConditionUtils.getIncludeQueryField(ESConstant.CONTENT_SIMHASH
- , ESConstant.TITLE_SIMHASH
- , ESConstant.TITLE
- , ESConstant.AUTHOR
- , ESConstant.CONTENT
- , ESConstant.SOURCE
- , ESConstant.PUBTIME
- , ESConstant.DOC_TYPE
- , ESConstant.CHANNEL
- , ESConstant.URL
- , ESConstant.SYS_SENTIMENT
- , ESConstant.DOC_ID
- , ESConstant.DATA_ID
- , ESConstant.SYS_ABSTRACT
- , ESConstant.COMMENT_SCORE
- , ESConstant.ATTITUDES_COUNT
- , ESConstant.COMMENT_ID
- , ESConstant.COMMENTS_COUNT
- , ESConstant.CRAWLTIME
- , ESConstant.ISDOWNLOAD
- , ESConstant.FILEPATH
- , ESConstant.VIDEOURL);
-
- // ID 查询字段
- public static final String[] FIELD_ID_LIST = {
- ESConstant.DATA_ID,
- ESConstant.DOC_ID};
-
- public static final String[] FIELD_DATA = EsQueryConditionUtils.getIncludeQueryField(
- ESConstant.AVATAR ,
- ESConstant.AUTHOR,
- ESConstant.DOC_ID ,
- ESConstant.PUBTIME ,
- ESConstant.SOURCE ,
- ESConstant.URL ,
- ESConstant.QUOTE_COUNT ,
- ESConstant.COMMENTS_COUNT ,
- ESConstant.ATTITUDES_COUNT ,
- ESConstant.TITLE ,
- ESConstant.CONTENT ,
- ESConstant.CHANNEL ,
- ESConstant.DOC_TYPE ,
- ESConstant.CRAWLTIME ,
- ESConstant.SYS_SENTIMENT);
-
- // 总体分析要用的字段
- public static final String[] FIELD_LIST_ANALYSIS =
- EsQueryConditionUtils.getIncludeQueryField(ESConstant.PUBTIME
- , ESConstant.CRAWLTIME
- , ESConstant.TITLE_SIMHASH
- , ESConstant.SCORE_FROM_SOURCE
- , ESConstant.DOC_ID
- , ESConstant.DOC_TYPE
- , ESConstant.HL_KEYWORDS
- , ESConstant.SOURCE
- , ESConstant.SYS_SENTIMENT
- , ESConstant.CONTENT_TAG
- , ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_NAME
- , ESConstant.EMOTION_ENTRY + ESConstant.DEFAULT_JOIN_SYMBOL + ESConstant.EMOTION_VALUE
- );
-
-
- // 词云查询字段
- public static final String[] FIELD_CLOUD_ANALYSIS= EsQueryConditionUtils.getIncludeQueryField(
- // 所有
- ESConstant.HL_KEYWORDS
- // 地名
- , ESConstant.PLACES
- // 话题
- , ESConstant.HASH_TAG
- // 评价
- , ESConstant.OPINIONS
- // 表情
- , ESConstant.EXPRESSION
- // ID
- , ESConstant.DOC_ID
- );
-
-
-}
diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ScoreFactorEntity.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ScoreFactorEntity.java
deleted file mode 100644
index 2a05013..0000000
--- a/cl_search_api/src/main/java/com/bfd/mf/common/util/constants/ScoreFactorEntity.java
+++ /dev/null
@@ -1,66 +0,0 @@
-package com.bfd.mf.common.util.constants;
-
-import java.io.Serializable;
-
-/**
- * Created by Changquanyou on 2017/7/12.
- * some scores of the factor
- */
-
-public class ScoreFactorEntity implements Serializable {
- private static final long serialVersionUID = 2322127803773558957L;
- /**Es 默认文档评分*/
- private Float tfidfScore;
-
- /**相似文章量得分*/
- private Float similarityNumber;
-
- /**情感标签得分*/
- private Double sentimentScore;
-
- /**网站排名得分*/
- private Float alexaScore;
-
- /**渠道得分*/
- private Float channelScore;
-
- public Float getTfidfScore() {
- return tfidfScore;
- }
-
- public void setTfidfScore(Float tfidfScore) {
- this.tfidfScore = tfidfScore;
- }
-
- public Float getSimilarityNumber() {
- return similarityNumber;
- }
-
- public void setSimilarityNumber(Float similarityNumber) {
- this.similarityNumber = similarityNumber;
- }
-
- public Double getSentimentScore() {
- return sentimentScore;
- }
-
- public void setSentimentScore(Double sentimentScore) {
- this.sentimentScore = sentimentScore;
- }
-
- public Float getAlexaScore() {
- return alexaScore;
- }
-
- public void setAlexaScore(Float alexaScore) {
- this.alexaScore = alexaScore;
- }
-
- public Float getChannelScore() {
- return channelScore;
- }
-
- public void setChannelScore(Float channelScore) {
- this.channelScore = channelScore;
- }
-}
diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/enums/CacheEnums.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/enums/CacheEnums.java
deleted file mode 100644
index 80fbddd..0000000
--- a/cl_search_api/src/main/java/com/bfd/mf/common/util/enums/CacheEnums.java
+++ /dev/null
@@ -1,308 +0,0 @@
-package com.bfd.mf.common.util.enums;
-
-
-public class CacheEnums {
- public enum TaskEventType {
- change_doc(0, "doc变更"),
- change_subject(1, "话题变更"),
- change_cache_swich(2, "开启/关闭缓存开关"),
- change_display_duration(3, "修改最大显示时长"),
- change_white_list(4, "白名单变更"),
- change_trace_back_duration(5, "修改创建之日起最大追溯时长");
-
- private int code;
- private String name;
- TaskEventType(int code, String name) {
- this.code = code;
- this.name = name;
- }
-
- public Integer getCode() {
- return code;
- }
-
- public String getName() {
- return name;
- }
-
- public static TaskEventType getByCode(int code) {
- for (TaskEventType taskEventType : TaskEventType.values()) {
- if (taskEventType.getCode().equals(code)) {
- return taskEventType;
- }
- }
- throw new IllegalArgumentException("code error");
- }
- }
-
- public enum TaskBatchType {
- regular(0, "定时"),
- rebuild(1, "重构"),
- recover(2, "恢复");
-
- private int code;
- private String name;
- TaskBatchType(int code, String name){
- this.code = code;
- this.name = name;
- }
-
- public Integer getCode() {
- return code;
- }
-
- public String getName() {
- return name;
- }
-
- public static TaskBatchType getByCode(int code) {
- for (TaskBatchType taskBatchType : TaskBatchType.values()) {
- if (taskBatchType.getCode().equals(code)) {
- return taskBatchType;
- }
- }
- throw new IllegalArgumentException("code error");
- }
- }
-
- public enum TaskOperation {
- delete(0, "删除"),
- create(1, "创建"),
- update(2, "修改");
-
- private int code;
- private String name;
- TaskOperation(int code, String name){
- this.code = code;
- this.name = name;
- }
-
- public Integer getCode() {
- return code;
- }
-
- public String getName() {
- return name;
- }
-
- public static TaskOperation getByCode(int code) {
- for (TaskOperation taskOperation : TaskOperation.values()) {
- if (taskOperation.getCode().equals(code)) {
- return taskOperation;
- }
- }
- throw new IllegalArgumentException("code error");
- }
- }
-
- public enum CacheSwitch {
- close(0, "关闭"),
- open(1, "开启"),
- closeAndSearch(2, "关闭缓存且直接查询数据");
-
- private int code;
- private String name;
- CacheSwitch(int code, String name){
- this.code = code;
- this.name = name;
- }
-
- public Integer getCode() {
- return code;
- }
-
- public String getName() {
- return name;
- }
-
- public static CacheSwitch getByCode(int code) {
- for (CacheSwitch cacheSwitch : CacheSwitch.values()) {
- if (cacheSwitch.getCode().equals(code)) {
- return cacheSwitch;
- }
- }
- throw new IllegalArgumentException("code error");
- }
- }
-
- public enum CacheRecalculateStatus {
- stable(0, "稳定"),
- rebuild(1, "重构");
-
- private int code;
- private String name;
- CacheRecalculateStatus(int code, String name){
- this.code = code;
- this.name = name;
- }
-
- public Integer getCode() {
- return code;
- }
-
- public String getName() {
- return name;
- }
-
- public static CacheRecalculateStatus getByCode(int code) {
- for (CacheRecalculateStatus cacheRecalculateStatus : CacheRecalculateStatus.values()) {
- if (cacheRecalculateStatus.getCode().equals(code)) {
- return cacheRecalculateStatus;
- }
- }
- throw new IllegalArgumentException("code error");
- }
- }
-
- public enum CacheRefreshStatus {
- stable(0, "稳定"),
- rebuild(1, "重构");
-
- private int code;
- private String name;
- CacheRefreshStatus(int code, String name){
- this.code = code;
- this.name = name;
- }
-
- public Integer getCode() {
- return code;
- }
-
- public String getName() {
- return name;
- }
-
- public static CacheRefreshStatus getByCode(int code) {
- for (CacheRefreshStatus cacheRefreshStatus : CacheRefreshStatus.values()) {
- if (cacheRefreshStatus.getCode().equals(code)) {
- return cacheRefreshStatus;
- }
- }
- throw new IllegalArgumentException("code error");
- }
- }
-
- public enum UpdateSubject {
- no(0, "不更新"),
- yes(1, "更新");
-
- private int code;
- private String name;
- UpdateSubject(int code, String name){
- this.code = code;
- this.name = name;
- }
-
- public Integer getCode() {
- return code;
- }
-
- public String getName() {
- return name;
- }
-
- public static UpdateSubject getByCode(int code) {
- for (UpdateSubject updateSubject : UpdateSubject.values()) {
- if (updateSubject.getCode().equals(code)) {
- return updateSubject;
- }
- }
- throw new IllegalArgumentException("code error");
- }
- }
-
- public static final Integer UpdateSubjectNumLeft = 60;
-
- public enum TaskSubjectStatus{
- kill(-2, "杀死"),
- fail(-1, "失败"),
- processing(0, "进行中"),
- success(1, "成功"),
- recover(2, "已恢复");
-
- private int code;
- private String name;
- TaskSubjectStatus(int code, String name){
- this.code = code;
- this.name = name;
- }
-
- public Integer getCode() {
- return code;
- }
-
- public String getName() {
- return name;
- }
-
- public static TaskSubjectStatus getByCode(int code) {
- for (TaskSubjectStatus taskSubjectStatus : TaskSubjectStatus.values()) {
- if (taskSubjectStatus.getCode().equals(code)) {
- return taskSubjectStatus;
- }
- }
- throw new IllegalArgumentException("code error");
- }
- }
-
- public enum TaskHandle {
- no(0, "不处理"),
- yes(1, "需处理");
-
- private int code;
- private String name;
- TaskHandle(int code, String name){
- this.code = code;
- this.name = name;
- }
-
- public Integer getCode() {
- return code;
- }
-
- public String getName() {
- return name;
- }
-
- public static TaskHandle getByCode(int code) {
- for (TaskHandle taskHandle : TaskHandle.values()) {
- if (taskHandle.getCode().equals(code)) {
- return taskHandle;
- }
- }
- throw new IllegalArgumentException("code error");
- }
- }
-
- public enum TaskBatchStage{
- undo(-1, "未执行"),
- start(0, "开始执行"),
- childrenCreate(1, "开始创建子任务"),
- childrenFinish(2, "创建子任务完毕");
-
- private int code;
- private String name;
- TaskBatchStage(int code, String name){
- this.code = code;
- this.name = name;
- }
-
- public Integer getCode() {
- return code;
- }
-
- public String getName() {
- return name;
- }
-
- public static TaskBatchStage getByCode(int code) {
- for (TaskBatchStage taskBatchStage : TaskBatchStage.values()) {
- if (taskBatchStage.getCode().equals(code)) {
- return taskBatchStage;
- }
- }
- throw new IllegalArgumentException("code error");
- }
- }
-}
diff --git a/cl_search_api/src/main/java/com/bfd/mf/common/util/enums/DataExportStatus.java b/cl_search_api/src/main/java/com/bfd/mf/common/util/enums/DataExportStatus.java
deleted file mode 100644
index 074b16f..0000000
--- a/cl_search_api/src/main/java/com/bfd/mf/common/util/enums/DataExportStatus.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright (C) 2016 Baifendian Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *