@ -3,16 +3,14 @@ package com.bfd.mf.entity.mysql;
/ / import com.bfd.crawler.utils.JsonUtils ; / / import com.bfd.crawler.utils.JsonUtils ;
import com.bfd.crawler.utils.JsonUtils ; import com.bfd.crawler.utils.JsonUtils ;
import com.bfd.mf.datasave.listen.DataSaveManager ;
import com.bfd.mf.datasave.tools.Constants ; import com.bfd.mf.datasave.tools.Constants ;
import com.bfd.mf.datasave.tools.DBUtil ; import com.bfd.mf.datasave.tools.DBUtil ;
import com.bfd.mf.datasave.tools.DateUtil ; import com.bfd.mf.datasave.tools.DateUtil ;
import com.bfd.mf.datasave.tools.WriteMethod ;
import crawler.open.util.RedisUtil ; import crawler.open.util.RedisUtil ;
import org.apache.log4j.Logger ; import org.apache.log4j.Logger ;
import java.text.SimpleDateFormat ;
import java.util.* ; import java.util.* ;
import java.util.concurrent.ArrayBlockingQueue ;
import static com.bfd.mf.entity.mysql.Tasklimit.subjectTasktimelimiit ; import static com.bfd.mf.entity.mysql.Tasklimit.subjectTasktimelimiit ;
import static com.bfd.mf.entity.mysql.Userlimit.subjectuserlimiit ; import static com.bfd.mf.entity.mysql.Userlimit.subjectuserlimiit ;
@ -186,171 +184,308 @@ public class SubjectTask implements Runnable {
@Override @Override
public void run ( ) { public void run ( ) {
while ( true ) {
subjectTaskMap . clear ( ) ;
Userlimit . loaduser ( ) ;
Tasklimit . loadTask ( ) ;
long updatetime = new Date ( ) . getTime ( ) / 1000 - 30000 ;
/ / List < Map < String , Object > > subjectTaskList = DBUtil . getInstance ( "db_stat" ) . query ( "select cs.del,ct.external_id, ct.subject_id, ct.id, ct.cid, ct.crawl_data_flag,cs.kafka_switch,cs.kafka_addr,cs.go_fast_addr,cs.kafka_topic,cs.go_fast_switch from cl_subject cs Join cl_task ct on(ct.subject_id=cs.id)where (ct.crawl_status=1 or ct.crawl_status=3) and ct.del=0 ;" ) ; ct . app_id = cs . app_id and
/ / String time = DateUtil . getDate ( ) ;
/ / System . out . println ( time ) ;
System . out . println ( "结束时间" + updatetime ) ;
List < Map < String , Object > > subjectTaskList = DBUtil . getInstance ( "db_stat_alltask" ) . query ( "select ct.crawl_content_key,ct.create_user_id,ct.app_id,cs.del,ct.external_id, ct.subject_id, ct.id, ct.cid, ct.crawl_data_flag,cs.kafka_switch,cs.kafka_addr,cs.go_fast_addr,cs.kafka_topic,cs.go_fast_switch from cl_subject cs Join cl_task ct on(ct.subject_id=cs.id) where (ct.crawl_status=1 ) and ct.del=0 and ct.app_id=cs.app_id and ct.cid!=\"\" and unix_timestamp(ct.update_time)>'" + updatetime + "' and ct.crawl_data_flag like '%气象侦察机%'order by ct.update_time desc;" ) ;
System . out . println ( subjectTaskList . size ( ) ) ;
if ( subjectTaskList . size ( ) > 0 ) {
String key = "" ;
for ( Map < String , Object > subjectTask : subjectTaskList ) { / / { subject_id = 10222 , name = 我是张三 , task_id = 188 , id = 71 , crawl_data_flag = aaa }
String keytwo = "" ;
if ( subjectTask . get ( "cid" ) . equals ( "Tmall" ) ) {
key = subjectTask . get ( "cid" ) + "#####" + subjectTask . get ( "crawl_data_flag" ) ;
keytwo = "Taobao" + "#####" + subjectTask . get ( "crawl_data_flag" ) ;
}
else if ( subjectTask . get ( "cid" ) . equals ( "Taobao" ) ) {
key = subjectTask . get ( "cid" ) + "#####" + subjectTask . get ( "crawl_data_flag" ) ;
keytwo = "Tmall" + "#####" + subjectTask . get ( "crawl_data_flag" ) ;
}
else {
key = subjectTask . get ( "cid" ) + "#####" + subjectTask . get ( "crawl_data_flag" ) ;
}
Map < String , String > value = new HashMap < > ( ) ;
List < Map < String , String > > valueList = new ArrayList < > ( ) ;
String v_subject_id = "" ;
String v_go_fast_addr = "" ;
String v_kafka_switch = "" ;
String v_kafka_addr = "" ;
String v_task_id = "" ;
String v_external_id = "" ;
String v_go_fast_switch = "" ;
String v_kafka_topic = "" ;
String v_status = "" ;
String v_del = "" ;
String v_create_user_id = "" ;
String v_ocr = "0" ;
String v_trans = "0" ;
String v_crawl_content_key = "" ;
if ( null ! = subjectTask . get ( "subject_id" ) ) {
v_subject_id = subjectTask . get ( "subject_id" ) . toString ( ) ;
}
if ( null ! = subjectTask . get ( "crawl_content_key" ) ) {
v_crawl_content_key = subjectTask . get ( "crawl_content_key" ) . toString ( ) ;
}
if ( null ! = subjectTask . get ( "go_fast_addr" ) ) {
v_go_fast_addr = subjectTask . get ( "go_fast_addr" ) . toString ( ) ;
}
if ( null ! = subjectTask . get ( "kafka_addr" ) ) {
v_kafka_addr = subjectTask . get ( "kafka_addr" ) . toString ( ) ;
}
if ( null ! = subjectTask . get ( "kafka_switch" ) ) {
v_kafka_switch = subjectTask . get ( "kafka_switch" ) . toString ( ) ;
}
if ( null ! = subjectTask . get ( "id" ) ) {
v_task_id = subjectTask . get ( "id" ) . toString ( ) ;
}
if ( null ! = subjectTask . get ( "external_id" ) ) {
v_external_id = subjectTask . get ( "external_id" ) . toString ( ) ;
}
if ( null ! = subjectTask . get ( "go_fast_switch" ) ) {
v_go_fast_switch = subjectTask . get ( "go_fast_switch" ) . toString ( ) ;
}
if ( null ! = subjectTask . get ( "kafka_topic" ) ) {
v_kafka_topic = subjectTask . get ( "kafka_topic" ) . toString ( ) ;
}
/ / if ( null ! = subjectTask . get ( "status" ) ) {
/ / v_status = subjectTask . get ( "status" ) . toString ( ) ;
/ / }
if ( null ! = subjectTask . get ( "del" ) ) {
v_del = subjectTask . get ( "del" ) . toString ( ) ;
}
if ( null ! = subjectTask . get ( "create_user_id" ) ) {
v_create_user_id = subjectTask . get ( "create_user_id" ) . toString ( ) ;
}
value . put ( "subject_id" , v_subject_id ) ;
value . put ( "go_fast_addr" , v_go_fast_addr ) ;
value . put ( "export_to_kafka" , v_kafka_switch ) ;
value . put ( "kafka_addr" , v_kafka_addr ) ;
/ / value . put ( "task_id" , v_task_id ) ;
value . put ( "external_id" , v_external_id ) ;
value . put ( "go_fast_switch" , v_go_fast_switch ) ;
value . put ( "kafka_topic" , v_kafka_topic ) ;
/ / value . put ( "status" , v_status ) ; / / 专题的状态
value . put ( "del" , v_del ) ; / / 专题的状态
value . put ( "appid" , subjectTask . get ( "app_id" ) . toString ( ) ) ;
value . put ( "crawl_content_key" , v_crawl_content_key ) ;
/ / System . out . print ( v_external_id + "external_id" ) ;
String newkey = key . toLowerCase ( ) ;
String userkey = newkey + "#####" + subjectTask . get ( "app_id" ) . toString ( ) . toLowerCase ( ) ;
while ( true ) try {
{
/ / subjectTaskMap . clear ( ) ;
Userlimit . loaduser ( ) ;
Tasklimit . loadTask ( ) ;
long updatetime = new Date ( ) . getTime ( ) / 1000 - 15 ;
SimpleDateFormat datetime = new SimpleDateFormat ( "yyyy-MM-dd HH:mm:ss" ) ;
String now = datetime . format ( Calendar . getInstance ( ) . getTime ( ) ) ;
System . out . println ( updatetime + now ) ;
long starttime = new Date ( ) . getTime ( ) / 1000 - 20L ;
if ( disposeCrawldataflag ( "Tasktimelimiit" ) ) {
starttime = Long . parseLong ( RedisUtil . get ( "Tasktimelimiit" , 9 ) ) ;
starttime = starttime - 5 ;
RedisUtil . set ( "Tasktimelimiit" , String . valueOf ( updatetime ) , 9 ) ;
} else {
System . out . println ( "第一次写入" ) ;
RedisUtil . set ( "Tasktimelimiit" , String . valueOf ( updatetime ) , 9 ) ;
}
System . out . println ( System . currentTimeMillis ( ) + "开始mysql执行的时间" ) ;
List < Map < String , Object > > subjectTaskendList = DBUtil . getInstance ( "db_stat_alltask" ) . query ( "select ct.crawl_content_key,ct.create_user_id,ct.app_id,cs.del,ct.crawl_start_time,ct.crawl_end_time,ct.external_id, ct.subject_id, ct.id, ct.cid, ct.crawl_data_flag,cs.kafka_switch,cs.kafka_addr,cs.go_fast_addr,cs.kafka_topic,cs.go_fast_switch from cl_subject cs Join cl_task ct on(ct.subject_id=cs.id) where (ct.crawl_status=1 ) and ct.del=0 and ct.cid!=\"\" and ct.app_id!='bw01' and unix_timestamp(ct.start_time)>'" + starttime + "';" ) ;
/ / 组装时间的参数
if ( subjectTasktimelimiit . containsKey ( userkey ) ) {
List < Map < String , String > > timelist = subjectTasktimelimiit . get ( userkey ) ;
if ( timelist . size ( ) = = 1 ) {
for ( Map < String , String > subjectTasktime : timelist ) {
value . put ( "maxtime" , subjectTasktime . get ( "max_time" ) . toString ( ) ) ;
value . put ( "mintime" , subjectTasktime . get ( "min_time" ) . toString ( ) ) ;
}
} else {
for ( Map < String , String > subjectTasktime : timelist ) {
String subject_id = subjectTasktime . get ( "subject_id" ) . toString ( ) ;
if ( v_subject_id . equals ( subject_id ) ) {
value . put ( "maxtime" , subjectTasktime . get ( "max_time" ) . toString ( ) ) ;
value . put ( "mintime" , subjectTasktime . get ( "min_time" ) . toString ( ) ) ;
}
}
}
/ / List < Map < String , Object > > subjectTaskendList = DBUtil . getInstance ( "db_stat_alltask" ) . query ( "select ct.crawl_content_key,ct.create_user_id,ct.app_id,cs.del,ct.external_id, ct.subject_id, ct.id, ct.cid, ct.crawl_data_flag,cs.kafka_switch,cs.kafka_addr,cs.go_fast_addr,cs.kafka_topic,cs.go_fast_switch from cl_subject cs Join cl_task ct on(ct.subject_id=cs.id) where (ct.crawl_status=1 ) and ct.del=0 and ct.cid!=\"\" and crawl_data_flag like '%http://xhslink.com/RLmh8p%';" ) ;
System . out . println ( System . currentTimeMillis ( ) + "执行mysql任务结束的时间" ) ;
if ( subjectTaskendList . size ( ) > 0 ) {
System . out . println ( subjectTaskendList . size ( ) + "任务的大小" ) ;
for ( Map < String , Object > subjectTask : subjectTaskendList ) {
String key = "" ;
String keytwo = "" ;
if ( subjectTask . get ( "cid" ) . equals ( "Tmall" ) ) {
key = subjectTask . get ( "cid" ) + "#####" + subjectTask . get ( "crawl_data_flag" ) ;
keytwo = "Taobao" + "#####" + subjectTask . get ( "crawl_data_flag" ) ;
}
else if ( subjectTask . get ( "cid" ) . equals ( "Taobao" ) ) {
key = subjectTask . get ( "cid" ) + "#####" + subjectTask . get ( "crawl_data_flag" ) ;
keytwo = "Tmall" + "#####" + subjectTask . get ( "crawl_data_flag" ) ;
}
else {
key = subjectTask . get ( "cid" ) + "#####" + subjectTask . get ( "crawl_data_flag" ) ;
}
Map < String , String > value = new HashMap < > ( ) ;
List < Map < String , String > > valueList = new ArrayList < > ( ) ;
String v_subject_id = "" ;
String v_go_fast_addr = "" ;
String v_kafka_switch = "" ;
String v_kafka_addr = "" ;
String v_task_id = "" ;
String v_external_id = "" ;
String v_go_fast_switch = "" ;
String v_kafka_topic = "" ;
String v_status = "" ;
String v_del = "" ;
String v_create_user_id = "" ;
String v_ocr = "0" ;
String v_trans = "0" ;
String v_crawl_content_key = "" ;
String appid = "" ;
if ( null ! = subjectTask . get ( "subject_id" ) ) {
v_subject_id = subjectTask . get ( "subject_id" ) . toString ( ) ;
}
if ( null ! = subjectTask . get ( "crawl_content_key" ) ) {
v_crawl_content_key = subjectTask . get ( "crawl_content_key" ) . toString ( ) ;
}
if ( null ! = subjectTask . get ( "go_fast_addr" ) ) {
v_go_fast_addr = subjectTask . get ( "go_fast_addr" ) . toString ( ) ;
}
if ( null ! = subjectTask . get ( "kafka_addr" ) ) {
v_kafka_addr = subjectTask . get ( "kafka_addr" ) . toString ( ) ;
}
if ( null ! = subjectTask . get ( "kafka_switch" ) ) {
v_kafka_switch = subjectTask . get ( "kafka_switch" ) . toString ( ) ;
}
if ( null ! = subjectTask . get ( "id" ) ) {
v_task_id = subjectTask . get ( "id" ) . toString ( ) ;
}
if ( null ! = subjectTask . get ( "external_id" ) ) {
v_external_id = subjectTask . get ( "external_id" ) . toString ( ) ;
}
if ( null ! = subjectTask . get ( "go_fast_switch" ) ) {
v_go_fast_switch = subjectTask . get ( "go_fast_switch" ) . toString ( ) ;
}
if ( null ! = subjectTask . get ( "kafka_topic" ) ) {
v_kafka_topic = subjectTask . get ( "kafka_topic" ) . toString ( ) ;
}
if ( null ! = subjectTask . get ( "del" ) ) {
v_del = subjectTask . get ( "del" ) . toString ( ) ;
}
if ( null ! = subjectTask . get ( "create_user_id" ) ) {
v_create_user_id = subjectTask . get ( "create_user_id" ) . toString ( ) ;
}
if ( null ! = subjectTask . get ( "app_id" ) ) {
appid = subjectTask . get ( "app_id" ) . toString ( ) ;
}
}
/ / / / 用户的权限
if ( subjectuserlimiit . containsKey ( v_create_user_id ) ) {
Map < String , Object > permission = ( Map < String , Object > ) subjectuserlimiit . get ( v_create_user_id ) ;
v_ocr = permission . get ( "is_ocr" ) . toString ( ) ;
v_trans = permission . get ( "is_trans" ) . toString ( ) ;
}
value . put ( "is_ocr" , v_ocr ) ;
value . put ( "is_trans" , v_trans ) ;
/ / 组装相同任务的任务id
if ( subjectTaskMap . containsKey ( newkey ) ) {
valueList = subjectTaskMap . get ( newkey ) ;
for ( Map < String , String > valuetask : valueList ) {
String task = valuetask . get ( "task_id" ) + "," + v_task_id ;
valuetask . put ( "task_id" , task ) ;
value . put ( "task_id" , task ) ;
}
valueList . add ( value ) ;
} else {
value . put ( "task_id" , v_task_id ) ;
valueList . add ( value ) ;
}
value . put ( "appid" , appid ) ;
value . put ( "subject_id" , v_subject_id ) ;
value . put ( "go_fast_addr" , v_go_fast_addr ) ;
value . put ( "export_to_kafka" , v_kafka_switch ) ;
value . put ( "kafka_addr" , v_kafka_addr ) ;
/ / value . put ( "task_id" , v_task_id ) ;
value . put ( "external_id" , v_external_id ) ;
value . put ( "go_fast_switch" , v_go_fast_switch ) ;
value . put ( "kafka_topic" , v_kafka_topic ) ;
value . put ( "del" , v_del ) ; / / 专题的状态
value . put ( "crawl_content_key" , v_crawl_content_key ) ;
value . put ( "maxtime" , subjectTask . get ( "crawl_end_time" ) . toString ( ) ) ;
value . put ( "mintime" , subjectTask . get ( "crawl_start_time" ) . toString ( ) ) ;
String newkey = key . toLowerCase ( ) ;
String userkey = newkey ;
/ / 组装时间的参数
try {
/ / if ( ! subjectTasktimelimiit . containsKey ( userkey ) ) { / / 如果无最大和最小时间
/ / Tasklimit . loadTask ( ) ;
/ / }
/ / List < Map < String , String > > timelist = subjectTasktimelimiit . get ( userkey ) ;
/ / if ( timelist . size ( ) = = 1 ) {
/ / for ( Map < String , String > subjectTasktime : timelist ) {
/ / value . put ( "maxtime" , subjectTasktime . get ( "max_time" ) . toString ( ) ) ;
/ / value . put ( "mintime" , subjectTasktime . get ( "min_time" ) . toString ( ) ) ;
/ / }
/ / } else {
/ / for ( Map < String , String > subjectTasktime : timelist ) {
/ / String subject_id = subjectTasktime . get ( "subject_id" ) . toString ( ) ;
/ / if ( v_subject_id . equals ( subject_id ) ) {
/ / value . put ( "maxtime" , subjectTasktime . get ( "max_time" ) . toString ( ) ) ;
/ / value . put ( "mintime" , subjectTasktime . get ( "min_time" ) . toString ( ) ) ;
/ / }
/ / }
/ / }
} catch ( Exception e ) {
System . out . print ( "获取时间失败了" + subjectTask . get ( "crawl_data_flag" ) ) ;
value . put ( "maxtime" , "1735660800000" ) ;
value . put ( "mintime" , "0" ) ;
e . printStackTrace ( ) ;
}
/ / / / 用户的权限
if ( subjectuserlimiit . containsKey ( v_create_user_id ) ) {
Map < String , Object > permission = ( Map < String , Object > ) subjectuserlimiit . get ( v_create_user_id ) ;
v_ocr = permission . get ( "is_ocr" ) . toString ( ) ;
v_trans = permission . get ( "is_trans" ) . toString ( ) ;
}
value . put ( "is_ocr" , v_ocr ) ;
value . put ( "is_trans" , v_trans ) ;
value . put ( "task_id" , v_task_id ) ;
valueList . add ( value ) ;
Set keysSet = new HashSet ( ) ;
/ / if ( subjectTaskMap . containsKey ( newkey ) ) {
/ / valueList = subjectTaskMap . get ( newkey ) ;
/ / for ( Map < String , String > valuetask : valueList ) {
/ / String task = valuetask . get ( "task_id" ) + "," + v_task_id ;
/ / valuetask . put ( "task_id" , task ) ;
/ / value . put ( "task_id" , task ) ;
/ / }
/ / valueList . add ( value ) ;
/ / } else {
/ / value . put ( "task_id" , v_task_id ) ;
/ / valueList . add ( value ) ;
/ / }
if ( disposeCrawldataflag ( newkey ) ) { / / 判断redis中的key
String getsubjectList = RedisUtil . get ( newkey , 9 ) ;
try {
List < Map < String , String > > subjectList = ( List < Map < String , String > > ) JsonUtils . parseArray ( getsubjectList ) ;
List < Map < String , String > > redistList = new ArrayList < > ( ) ;
for ( Map < String , String > redissubjectTask : subjectList ) {
String resubject_id = redissubjectTask . get ( "subject_id" ) . toString ( ) ;
String reappid = redissubjectTask . get ( "appid" ) . toString ( ) ;
String taskid = redissubjectTask . get ( "task_id" ) . toString ( ) ;
String maxtime = "" ;
if ( redissubjectTask . containsKey ( "maxtime" ) ) {
maxtime = redissubjectTask . get ( "maxtime" ) . toString ( ) ;
}
if ( ! resubject_id . equals ( v_subject_id ) | | ! reappid . equals ( appid ) | | ! v_task_id . equals ( taskid ) ) {
redistList . add ( redissubjectTask ) ; ;
if ( keytwo . length ( ) > 0 ) {
String tmallnewkey = keytwo . toLowerCase ( ) ;
subjectTaskMap . put ( tmallnewkey , valueList ) ;
}
String redis = newkey + "@#@" + JsonUtils . toJSONString ( valueList ) ;
try {
Constants . getLineQueue ( ) . put ( redis ) ;
} catch ( InterruptedException e ) {
e . printStackTrace ( ) ;
}
}
valueList . addAll ( redistList ) ;
String redis = newkey + "@#@" + JsonUtils . toJSONString ( valueList ) ;
try {
System . out . println ( "写入redis是" + redis ) ;
Constants . getLineQueue ( ) . put ( redis ) ;
} catch ( InterruptedException e ) {
e . printStackTrace ( ) ;
}
/ / 天猫
if ( keytwo . length ( ) > 0 ) {
String tmallnewkey = keytwo . toLowerCase ( ) ;
String redis2 = tmallnewkey + "@#@" + JsonUtils . toJSONString ( valueList ) ;
try {
Constants . getLineQueue ( ) . put ( redis2 ) ;
} catch ( InterruptedException e ) {
e . printStackTrace ( ) ;
}
}
} catch ( Exception e ) {
log . error ( "获取redis中的任务失败" + getsubjectList ) ;
}
} else {
String redis = newkey + "@#@" + JsonUtils . toJSONString ( valueList ) ;
System . out . println ( redis + "不存在" ) ;
try {
Constants . getLineQueue ( ) . put ( redis ) ;
} catch ( InterruptedException e ) {
e . printStackTrace ( ) ;
}
if ( keytwo . length ( ) > 0 ) {
String tmallnewkey = keytwo . toLowerCase ( ) ;
String redis2 = tmallnewkey + "@#@" + JsonUtils . toJSONString ( valueList ) ;
try {
Constants . getLineQueue ( ) . put ( redis2 ) ;
} catch ( InterruptedException e ) {
e . printStackTrace ( ) ;
}
}
}
/ / subjectTaskMap . put ( newkey , valueList ) ;
/ / / / System . out . print ( valueList ) ;
/ /
/ / / / System . out . print ( JsonUtils . toJSONString ( subjectTaskMap ) ) ;
/ /
/ / Set keysSets = new HashSet ( ) ;
/ / for ( String newkeya : subjectTaskMap . keySet ( ) ) {
/ / List < Map < String , String > > newkeyvalue = subjectTaskMap . get ( newkeya ) ;
/ / List < Map < String , String > > setkeyvalue = new ArrayList < > ( ) ;
/ / for ( Map < String , String > redissubjectTask : newkeyvalue ) {
/ / String resubject_id = redissubjectTask . get ( "subject_id" ) . toString ( ) ;
/ / String reappid = redissubjectTask . get ( "appid" ) . toString ( ) ;
/ / String taskid = redissubjectTask . get ( "task_id" ) . toString ( ) ;
/ / String keys = resubject_id + reappid + newkeya + taskid ;
/ / RedisUtil . set ( keys , "1" , 10 ) ;
/ / / / int beforeSize = keysSets . size ( ) ;
/ / / / keysSets . add ( keys ) ;
/ / / / int afterSize = keysSets . size ( ) ;
/ / / / if ( afterSize = = beforeSize + 1 ) {
/ / / / setkeyvalue . add ( redissubjectTask ) ;
/ / / / }
/ / setkeyvalue . add ( redissubjectTask ) ;
/ / }
/ / String redis = newkeya + "@#@" + JsonUtils . toJSONString ( setkeyvalue ) ;
/ / System . out . println ( redis ) ;
/ / try {
/ / Constants . getLineQueue ( ) . put ( redis ) ;
/ / } catch ( InterruptedException e ) {
/ / e . printStackTrace ( ) ;
/ / }
/ / }
System . out . println ( System . currentTimeMillis ( ) + "任务组装结束的时间" ) ;
log . info ( "结束时间" + DateUtil . getcurr ( ) ) ;
log . info ( "当天任务的数量" + key + " ; data = " + subjectTaskendList . size ( ) ) ;
/ / } else {
/ / String delkey = subjectendtTask . get ( "cid" ) + "#####" + subjectendtTask . get ( "crawl_data_flag" ) ;
/ / log . info ( delkey . toLowerCase ( ) + "====del" ) ;
/ / RedisUtil . del ( delkey . toLowerCase ( ) , 10 ) ;
/ / }
}
} else {
Thread . sleep ( 1000 * 5 ) ;
}
}
} catch ( Exception e ) {
e . printStackTrace ( ) ;
}
}
private boolean disposeCrawldataflag ( String crawldataflag ) {
try {
if ( RedisUtil . exists ( crawldataflag , 9 ) ) { / / 先去 redis中查询是否存在 ， 不存直接忽略
String value = RedisUtil . get ( crawldataflag , 9 ) ;
if ( null ! = value & & ! ( "" ) . equals ( value ) ) {
return true ;
} }
/ / RedisUtil . set ( newkey , JsonUtils . toJSONString ( valueList ) , 10 ) ;
/ / System . out . println ( "结束时间" + DateUtil . getcurr ( ) ) ;
subjectTaskMap . put ( newkey , valueList ) ;
/ / System . out . println ( newkey ) ;
} else {
log . error ( "[datasave] exec >>> 灌数：该 crwaldataflag 在 Redis 中不存在！！！ keys = " + crawldataflag + " ; dbindex = " + 10 ) ;
return false ;
} }
System . out . println ( "结束时间" + DateUtil . getcurr ( ) ) ;
/ / System . out . println ( subjectTaskMap . size ( ) ) ;
log . info ( "当天任务的数量" + key + " ; data = " + subjectTaskMap . size ( ) ) ;
/ / SimpleDateFormat df = new SimpleDateFormat ( "yyyy-MM-dd HH:mm:ss" ) ; / / 设置日期格式
/ / System . out . println ( subjectTaskList . size ( ) ) ; / / new Date ( ) 为获取当前系统时间
/ / WriteMethod . writeMethod ( "0621test.txt" , JsonUtils . toJSONString ( subjectTaskMap ) ) ;
/ / System . out . println ( JsonUtils . toJSONString ( subjectTaskMap ) + "当前时间" + DateUtil . getcurr ( ) ) ;
return false ;
} catch ( Exception e ) {
e . printStackTrace ( ) ;
return false ;
} }
try {
Thread . sleep ( 3000 ) ;
} catch ( InterruptedException e ) {
e . printStackTrace ( ) ;
}
} }
public static void main ( String [ ] args ) {
String resubject_id = "1" ;
String v_subject_id = "1" ;
String reappid = "12" ;
String appid = "12" ;
String v_task_id = "12" ;
String taskid = "13" ;
if ( ! resubject_id . equals ( v_subject_id ) | | ! reappid . equals ( appid ) | | ! v_task_id . equals ( taskid ) ) {
System . out . println ( "11" ) ;
}
} }
} }
zhicheng.zhang
4 weeks ago