diff --git a/clean.py b/clean.py index 53c1cf5..4306443 100644 --- a/clean.py +++ b/clean.py @@ -1,5 +1,5 @@ import os -root_path = os.path.abspath(os.path.dirname(__file__)).split('telegram-crawler')[0] + "telegram-crawler" +root_path = os.path.abspath(os.path.dirname(__file__)).split('telegram_crawler')[0] + "telegram_crawler" import time from loguru import logger import shutil diff --git a/config.py b/config.py index 337af81..460f985 100644 --- a/config.py +++ b/config.py @@ -10,7 +10,7 @@ SESSION_NAME = "session_name" API_ID = 28340634 API_HASH = '5c54ebfc2729b32bc6f49e4b34747f47' SESSION_STRING = "1BVtsOG8Buw04Jkp3pNiWll0L2AzXwTGUCNpDnxPvUagdG7iLL_oaHoiepFG3LqF0gnZUe60-IXMn-MCGjQ_tTPjZP0VDeSl_0EmnKjZvaRxvX8pPKbmbLqZmvJjPmplEbel1CosRzCC-naBrnoE6vGetrzAvawzY8OA8_R5VNmGXCaCEPKumC9Safh7qLUntW70e2V4GYqRzwoYkZPdZzQKcfYufKrt3cfnTrCESE4_EDTRotGcqRm6IaAx9XLCrFsDwIPNJYf5ij6-Uzq8cx6qAD1IpSLy7hDehH1gfkM4N4Wem4LduJVlmHnV5Q4IHmiXrE2YAO4DTUpKV0_UMJRDDLkjB8XM=" -ENABLE_PROXY = True +ENABLE_PROXY = False if ENABLE_PROXY: PROXY = ("socks5", '127.0.0.1', 1080) else: @@ -39,11 +39,11 @@ TOPIC_DIC = { # "testtelegram": "test_telegram", "testtelegram": "test_telegram_1" } -TOPIC_ADDRESS = "" # kafka地址 +TOPIC_ADDRESS = "172.16.228.89:19092" # kafka地址 # ************************************************************** # 上传链接 -UPLOAD_URL = "http://x/upload" +UPLOAD_URL = "http://10.8.0.10:8081/group1/upload" # ************************************************************** # 日志 @@ -61,13 +61,13 @@ logger = MyLogger() # DB_FULL_NAME = "" class CRAWLER_DB_CONF(object): - DBHOST = '127.0.0.1' + DBHOST = '172.16.228.89' DBPORT = 3306 - DBUSER = 'root' - DBPWD = '1q2w3e4r' - DBNAME = 'tg_message' + DBUSER = 'crawl' + DBPWD = 'crawl123' + DBNAME = 'oversea_crawl_social' DBCHAR = 'utf8mb4' - DB_FULL_NAME = "tg_message" + DB_FULL_NAME = "oversea_crawl_social" TG_ROBOT_ACCOUNT_TABLE = 'tg_robot_account' diff --git a/test.py b/test.py index 420f1f8..f5944f1 100644 --- a/test.py +++ b/test.py @@ -6,7 +6,7 @@ from telethon.tl.functions.channels import JoinChannelRequest from tg_module.initial_group import join_channel from tg_utils.tg_model import GroupFunc -root_path = os.path.abspath(os.path.dirname(__file__)).split('telegram-crawler')[0] + "telegram-crawler" +root_path = os.path.abspath(os.path.dirname(__file__)).split('telegram_crawler')[0] + "telegram_crawler" sys.path.append(root_path) from utils.MysqlData import MysqlPoolClient @@ -156,7 +156,8 @@ async def main(session_string, api_id, api_hash, kp=None, message_topic=""): message = event.message media = event.media other_link = await get_extra_linked(message) # 获得超链接 - file_list, user_photo = await download_resources(message, media, client, sender) + # file_list, user_photo = await download_resources(message, media, client, sender) + file_list, user_photo = None, None message_text = message.message date = message.date replay = message.reply_to # MessageReplyHeader 对象 @@ -292,11 +293,11 @@ async def leave_group(session_string, api_id, api_hash): await client.disconnect() async def login(): - api_id = "28340634" - api_hash = "5c54ebfc2729b32bc6f49e4b34747f47" - data = {'id': 2, 'phone': '18846824798', 'api_id': '28340634', - 'api_hash': '5c54ebfc2729b32bc6f49e4b34747f47', - 'session_string': '1BVtsOMUBu22sAF4QtYuqDLre0vn4JxLy8utbzMvJ7kd6q6ZKNW_lIwUZEu38uPiRsz7uWiGK9gQURYPdMCLSjnWejTZ1pbdPT4kWSYfl4gN2iGb5G_Ib4mQpW4XgUErpiyIiCaTfG9ph59bz4Y-sw2rDhHSPVoL6PttS_OnQI5PKbVKSQLmO8A6OGoauNyf1_MlmPv1MdQUkA6Ep7fgXcKWKUUZ3YWkmIojWDzz02505D0Jcn-mQ3ED6zESToZafZsyj9Ktcf2NUzJA44DXMy2RcOfNjyyFMafl1kNrLAJ2uoz5vwzM0adw38AYhxVOH6VYbBm8ZnKf7T-9EQkwsXMKZL1FFJ8A=', + # api_id = "28340634" + # api_hash = "5c54ebfc2729b32bc6f49e4b34747f47" + data = {'id': 2, 'phone': '18846824798', 'api_id': '22955009', + 'api_hash': 'c5581d1ed5880f6e0b1734dad1d4a10d', + 'session_string': '1BVtsOIEBuwG1-0k5xGzlopl4G7ghAhBPorz1HcaSkfcuDEsYKSJKQ0nCLYbMTT7yplnfJHEYXR-rGY5FoEyrAYsW86obngGwxLDpl9b9IuGhxCDlFSo_O2AIPw3Duf8tc5DewfNGqZ7U8CbpEjFcpEaRRy23Z93DVZtcYHBLp6vLh5iLndKXanW4vxArJODjVklAKwxqDD5LGixvoeP5p9W1VJAeihJxqEl0UHL12dF4T7MYcdhW-ylA4NvCtgeIaqiVwZ1VuVyiyLNYaMrpZZfdmfOGkYapp-1ubYE8XKAQ8jau3XnWCLvk50w6L9DaWp8PSdQ7RRZf5G2swSyurHCK6quAVfA=', 'plateform': 'DeskTop', 'total_group': 274} session = StringSession(data["session_string"]) @@ -316,10 +317,10 @@ async def mul_account(): tasks = [] - api_id = "22955009" - api_hash = "c5581d1ed5880f6e0b1734dad1d4a10d" + api_id = "28279639" + api_hash = "dc7eb234661a51bfd723ecd201797e1a" # session_string = "demo1" - session_string = "1BVtsOIEBuwG1-0k5xGzlopl4G7ghAhBPorz1HcaSkfcuDEsYKSJKQ0nCLYbMTT7yplnfJHEYXR-rGY5FoEyrAYsW86obngGwxLDpl9b9IuGhxCDlFSo_O2AIPw3Duf8tc5DewfNGqZ7U8CbpEjFcpEaRRy23Z93DVZtcYHBLp6vLh5iLndKXanW4vxArJODjVklAKwxqDD5LGixvoeP5p9W1VJAeihJxqEl0UHL12dF4T7MYcdhW-ylA4NvCtgeIaqiVwZ1VuVyiyLNYaMrpZZfdmfOGkYapp-1ubYE8XKAQ8jau3XnWCLvk50w6L9DaWp8PSdQ7RRZf5G2swSyurHCK6quAVfA=" + session_string = "1BVtsOMUBu6HAZK23ZSobi4_zwJPrKdjckKtX3NzXA6mj26BaZTQ3fGlhKNke-A7unmk2mH4ESMi7mfQx2vvogezgj8C-W7VO2V_91vTIZv-crQ3yaQNMSwwEJaOYRmCr5SrLV1cILYP0bgP0o2Sr-KIP82Z0mDxfJycW0q65liB3W_nZMqiO2131QK8_Nq_MEYrG8uKj6rV8XQOcSTZYmu27_dHz_elbJMrJZvcFGCg3l7EtoZr7VlQDcUP_4wfoSHWGqOuipxnnmZbd97mZENrTsfPdJNdX_vI7QPRrK69p2mwmlXQ6Waj_6zKp3nHNQPSuK4Athiy6Fcle8TZ9BRQep8IryxA=" tasks.append(main(session_string, api_id, api_hash)) # tasks.append(login()) diff --git a/tg_module/initial_group.py b/tg_module/initial_group.py index b6a9297..cf2fc94 100644 --- a/tg_module/initial_group.py +++ b/tg_module/initial_group.py @@ -8,7 +8,7 @@ from telethon.tl.functions.messages import GetFullChatRequest, ImportChatInviteR from tg_utils.tg_api import send_messages -root_path = os.path.abspath(os.path.dirname(__file__)).split('telegram-crawler')[0] + "telegram-crawler" +root_path = os.path.abspath(os.path.dirname(__file__)).split('telegram_crawler')[0] + "telegram_crawler" sys.path.append(root_path) from tg_utils.tg_model import GroupFunc diff --git a/tg_module/receive_message.py b/tg_module/receive_message.py index 9b55946..550178d 100644 --- a/tg_module/receive_message.py +++ b/tg_module/receive_message.py @@ -1,6 +1,6 @@ # code=utf-8 import os, sys -root_path = os.path.abspath(os.path.dirname(__file__)).split('telegram-crawler')[0] + "telegram-crawler" +root_path = os.path.abspath(os.path.dirname(__file__)).split('telegram_crawler')[0] + "telegram_crawler" sys.path.append(root_path) from utils.MysqlData import MysqlPoolClient @@ -190,7 +190,7 @@ async def main(session_string, api_id, api_hash, kp=None, message_topic=""): async def mul_account(): client_mysql = MysqlPoolClient(CRAWLER_DB_CONF) - sql = f"select * from {TG_ROBOT_ACCOUNT_TABLE} order by update_time" + sql = f"select * from {TG_ROBOT_ACCOUNT_TABLE} where api_id!='28340634' order by update_time" results = client_mysql.getAll(sql) kp = SKafka(bootstrap_servers=TOPIC_ADDRESS) message_topic = TOPIC_DIC["testtelegram"]