zhicheng
/
caijitask
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

采集平台任务缓存

sitask
zhicheng.zhang 4 weeks ago
parent
baa63fad8c
commit
bde2a62c6b
2 changed files with 30 additions and 12 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 40
      .idea/workspace.xml
  2. 2
      cl_stream_datasave/src/main/java/com/bfd/mf/entity/mysql/SubjectTask.java

40
.idea/workspace.xml
View File

@ -9,13 +9,9 @@
<option name="autoReloadType" value="SELECTIVE" />
</component>
<component name="ChangeListManager">
<list default="true" id="3054eaf2-8522-4f3f-9b45-79fa527c3b8f" name="Default" comment="海外任务缓存">
<change beforePath="$PROJECT_DIR$/cl_offline_task/cl_offline_task.iml" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/cl_offline_task/pom.xml" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/cl_offline_task/src/main/java/com/bfd/mf/entity/Model.java" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/cl_offline_task/src/main/java/com/bfd/mf/entity/SubjectTask.java" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/cl_offline_task/src/main/java/com/bfd/mf/task/RunStart.java" beforeDir="false" />
<change beforePath="$PROJECT_DIR$/cl_offline_task/src/main/main4.iml" beforeDir="false" />
<list default="true" id="30c18d53-b779-429b-9705-58d485046a45" name="Default Changelist" comment="海外任务缓存">
<change beforePath="$PROJECT_DIR$/cl_stream_datasave/src/main/java/com/bfd/mf/datasave/tools/DataProcess.java" beforeDir="false" afterPath="$PROJECT_DIR$/cl_stream_datasave/src/main/java/com/bfd/mf/datasave/tools/DataProcess.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/cl_stream_datasave/src/main/java/com/bfd/mf/entity/mysql/SubjectTask.java" beforeDir="false" afterPath="$PROJECT_DIR$/cl_stream_datasave/src/main/java/com/bfd/mf/entity/mysql/SubjectTask.java" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
@ -81,7 +77,7 @@
</option>
<option name="RECENT_BRANCH_BY_REPOSITORY">
<map>
<entry key="$PROJECT_DIR$" value="sitask" />
<entry key="$PROJECT_DIR$" value="si" />
</map>
</option>
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
@ -410,7 +406,7 @@
<component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="3054eaf2-8522-4f3f-9b45-79fa527c3b8f" name="Default" comment="" />
<changelist id="" name="Default" comment="" />
<created>1641456920617</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
@ -540,6 +536,7 @@
<workItem from="1746596390545" duration="607000" />
<workItem from="1748499221032" duration="1243000" />
<workItem from="1748500471536" duration="1630000" />
<workItem from="1748502137860" duration="347000" />
</task>
<task id="LOCAL-00001" summary="海外任务缓存">
<created>1748499561964</created>
@ -576,7 +573,28 @@
<option name="project" value="LOCAL" />
<updated>1748501169875</updated>
</task>
<option name="localTasksCounter" value="6" />
<task id="LOCAL-00006" summary="海外任务缓存">
<created>1748502260846</created>
<option name="number" value="00006" />
<option name="presentableId" value="LOCAL-00006" />
<option name="project" value="LOCAL" />
<updated>1748502260846</updated>
</task>
<task id="LOCAL-00007" summary="海外任务缓存">
<created>1748502276509</created>
<option name="number" value="00007" />
<option name="presentableId" value="LOCAL-00007" />
<option name="project" value="LOCAL" />
<updated>1748502276509</updated>
</task>
<task id="LOCAL-00008" summary="海外任务缓存">
<created>1748502372901</created>
<option name="number" value="00008" />
<option name="presentableId" value="LOCAL-00008" />
<option name="project" value="LOCAL" />
<updated>1748502372901</updated>
</task>
<option name="localTasksCounter" value="9" />
<servers />
</component>
<component name="TimeTrackingManager">
@ -662,7 +680,7 @@
<entry key="branch">
<value>
<list>
<option value="si" />
<option value="sitask" />
</list>
</value>
</entry>

2
cl_stream_datasave/src/main/java/com/bfd/mf/entity/mysql/SubjectTask.java
View File

@ -203,7 +203,7 @@ public class SubjectTask implements Runnable {
RedisUtil.set("Tasktimelimiit", String.valueOf(updatetime), 10);
}
System.out.println(System.currentTimeMillis()+"开始mysql执行的时间");
List<Map<String, Object>> subjectTaskendList = DBUtil.getInstance("db_stat_alltask").query("select ct.crawl_content_key,ct.create_user_id,ct.app_id,cs.del,ct.crawl_start_time,ct.crawl_end_time,ct.external_id, ct.subject_id, ct.id, ct.cid, ct.crawl_data_flag,cs.kafka_switch,cs.kafka_addr,cs.go_fast_addr,cs.kafka_topic,cs.go_fast_switch from cl_subject cs Join cl_task ct on(ct.subject_id=cs.id) where (ct.crawl_status=1 ) and ct.del=0 and ct.cid!=\"\" and ct.app_id!='bw01' and unix_timestamp(ct.start_time)>'"+starttime+"';");
List<Map<String, Object>> subjectTaskendList = DBUtil.getInstance("db_stat_alltask").query("select ct.crawl_content_key,ct.create_user_id,ct.app_id,cs.del,ct.crawl_start_time,ct.crawl_end_time,ct.external_id, ct.subject_id, ct.id, ct.cid, ct.crawl_data_flag,cs.kafka_switch,cs.kafka_addr,cs.go_fast_addr,cs.kafka_topic,cs.go_fast_switch from cl_subject cs Join cl_task ct on(ct.subject_id=cs.id) where (ct.crawl_status=1 ) and ct.del=0 and ct.cid!=\"\" and unix_timestamp(ct.start_time)>'"+starttime+"';");
// List<Map<String, Object>> subjectTaskendList = DBUtil.getInstance("db_stat_alltask").query("select ct.crawl_content_key,ct.create_user_id,ct.app_id,cs.del,ct.external_id, ct.subject_id, ct.id, ct.cid, ct.crawl_data_flag,cs.kafka_switch,cs.kafka_addr,cs.go_fast_addr,cs.kafka_topic,cs.go_fast_switch from cl_subject cs Join cl_task ct on(ct.subject_id=cs.id) where (ct.crawl_status=1 ) and ct.del=0 and ct.cid!=\"\" and crawl_data_flag like '%http://xhslink.com/RLmh8p%';");
System.out.println(System.currentTimeMillis()+"执行mysql任务结束的时间");

Write Preview
Loading…
Cancel
Save