from requests_html import HTML, HTMLSession from retrying import retry from lottery import Lottery session = HTMLSession() @retry(stop_max_attempt_number=3) def _get_data(url, lottery_type): r = session.get(url) table_list = r.html.find("table.kj_tablelist02", first=True) draw_issue = table_list.find("td.td_title01 span.span_left strong")[0].text draw_date = table_list.find("td.td_title01 span.span_right")[0].text draw_code = table_list.find("div.ball_box01")[0].text.replace('\n', '') with Lottery(lottery_type=lottery_type) as lottery: last_id = lottery.insert(draw_issue, draw_date, draw_code) if last_id: print(f"issue:{draw_issue}数据写入完成。。。") else: print(f'issue:{draw_issue}已经存在') def get_data(url, lottery_type): try: _get_data(url, lottery_type) except Exception as e: print(e) print(url) print('异常出错重试后,依然报错') raise e class DataGetter(object): def __init__(self): self.session = HTMLSession() @retry(stop_max_attempt_number=3) def _get_data(self, issue, lottery_type='pls'): """ """ r = self.session.get(f"https://kaijiang.500.com/shtml/{lottery_type}/{issue}.shtml") if lottery_type == 'sfc': # 开奖期数 issue_elem = r.html.search('胜负彩(任九) 第 {} 期') draw_issue = HTML(html=issue_elem[0]).text # 胜负彩销量 sfc_sale_elem = r.html.search('本期胜负彩销量:{}元') sfc_sale = HTML(html=sfc_sale_elem[0]).text.replace(",", "") # 任九销量 rj_sale_elem = r.html.search('本期任九销量:{}元') rj_sale = HTML(html=rj_sale_elem[0]).text.replace(",", "") # 开奖结果 table_list = r.html.find("table.kj_tablelist02", first=True) draw_code = table_list.find("tr")[2].text.replace('\n', ' ') # 开奖日期 draw_date = table_list.find("td.td_title01 span.span_right")[0].text # 中奖详情 table_list_02 = r.html.find("table.kj_tablelist02")[1] award_1_elem = table_list_02.find("tr")[2] award_2_elem = table_list_02.find("tr")[3] rj_award_elem = table_list_02.find("tr")[4] award_1_num = award_1_elem.find("td")[1].text.replace(",", "") award_1_money = award_1_elem.find("td")[2].text.replace(",", "") award_2_num = award_2_elem.find("td")[1].text.replace(",", "") award_2_money = award_2_elem.find("td")[2].text.replace(",", "") r9_award_num = rj_award_elem.find("td")[1].text.replace(",", "") r9_award_money = rj_award_elem.find("td")[2].text.replace(",", "") with Lottery(lottery_type='sfc') as lottery: last_id = lottery.insert(draw_issue, draw_date, draw_code, sfc_sale=sfc_sale, rj_sale=rj_sale, award_1_num=award_1_num, award_1_money=award_1_money, award_2_num=award_2_num, award_2_money=award_2_money, r9_award_num=r9_award_num, r9_award_money=r9_award_money) if last_id: print(f"issue:{draw_issue}数据写入完成。。。") else: print(f'issue:{draw_issue}已经存在') else: table_list = r.html.find("table.kj_tablelist02", first=True) draw_issue = table_list.find("td.td_title01 span.span_left strong")[0].text draw_date = table_list.find("td.td_title01 span.span_right")[0].text draw_code = table_list.find("div.ball_box01")[0].text.replace('\n', '') with Lottery(lottery_type=lottery_type) as lottery: last_id = lottery.insert(draw_issue, draw_date, draw_code) if last_id: print(f"issue:{draw_issue}数据写入完成。。。") else: print(f'issue:{draw_issue}已经存在') def pls(self, issue): ''' 排列三 ''' try: return self._get_data(issue, lottery_type='pls') except Exception as e: print(e) def plw(self, issue): ''' 排列五 ''' try: return self._get_data(issue, lottery_type='plw') except Exception as e: print(e) def sd(self, issue): ''' 3D ''' try: return self._get_data(issue, lottery_type='sd') except Exception as e: print(e) def sfc(self, issue): ''' 胜负彩 ''' try: return self._get_data(issue, lottery_type='sfc') except Exception as e: print(e) if __name__ == '__main__': # issue = "23108" # url = f"https://kaijiang.500.com/shtml/pls/{issue}.shtml" # get_data(url, lottery_type='pls') getter = DataGetter() getter.plw('23124') # for i in range(1, 153): # try : # issue = f"{22}{i:03d}" # getter.sfc(issue=issue) # except Exception as e: # print(e) # continue