diff --git a/algorithm_datastructure/dict_tools.py b/algorithm_datastructure/dict_tools.py index 0ae1541..316c67a 100644 --- a/algorithm_datastructure/dict_tools.py +++ b/algorithm_datastructure/dict_tools.py @@ -4,6 +4,7 @@ class ObjectDict(dict): """from tornado.util.ObjectDict, access dict by dot""" + def __getattr__(self, key): try: return self[key] @@ -16,7 +17,6 @@ def __setattr__(self, key, value): import collections - class TransformedDict(collections.MutableMapping): """A dictionary that applies an arbitrary key-altering function before accessing the keys""" @@ -43,10 +43,12 @@ def __len__(self): def __keytransform__(self, key): return key + # pip install fronzendict class FronzenDict(TransformedDict): """immutable dict, init by a dict""" + def __init__(self, d): self.store = d @@ -57,4 +59,7 @@ def __setitem__(self, key, value): # since python3.3, you can use immutable dict import types -d_proxy = types.MappingProxyType(d) # can add new key/value by assignment but can not assign key alrady exists + +d = {1: 'hello', 2: 'world'} +d_proxy = types.MappingProxyType(d) # can add new key/value by assignment but can not assign key alrady exists +print(d_proxy) \ No newline at end of file diff --git a/autotest/curl_to_request.py b/autotest/curl_to_request.py index 41fcfe4..4e157b8 100644 --- a/autotest/curl_to_request.py +++ b/autotest/curl_to_request.py @@ -49,5 +49,5 @@ def parse_curl_str(s): if __name__ == '__main__': - s = """'curl http://pre3.papayamobile.com:1267/shoptimize/product/get_category_ids' -H 'Cookie: language=zh_CN; _gat=1; ppysid="PMs5bkeZejk5DZ9YuE+XRySRFW58DvWlgNYfI2YP+SQ="; save=true; email=fan_ll%40qq.com; password=6be037423107205d176a1d0d174402fac6b1dcc3d1b99b894fdbdf1d52ea9935; login_type=pmd; Hm_lvt_4f18dfc7029aaa9ff0b37ede9b128ef4=1463108257; Hm_lpvt_4f18dfc7029aaa9ff0b37ede9b128ef4=1463587952; _ga=GA1.2.2053141674.1463108258' -H 'Origin: http://pre3.papayamobile.com:1267' -H 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4' -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36' -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' -H 'Accept: application/json, text/javascript, */*; q=0.01' -H 'Referer: http://pre3.papayamobile.com:1267/shoptimize/campaign?type=website' -H 'X-Requested-With: XMLHttpRequest' -H 'Connection: keep-alive' --data 'product_type=0&account_id=2' --compressed""" + s = """curl 'http://pre3.papayamobile.com:1267/shoptimize/product/get_category_ids' -H 'Cookie: language=zh_CN; _gat=1; ppysid="PMs5bkeZejk5DZ9YuE+XRySRFW58DvWlgNYfI2YP+SQ="; save=true; email=fan_ll%40qq.com; password=6be037423107205d176a1d0d174402fac6b1dcc3d1b99b894fdbdf1d52ea9935; login_type=pmd; Hm_lvt_4f18dfc7029aaa9ff0b37ede9b128ef4=1463108257; Hm_lpvt_4f18dfc7029aaa9ff0b37ede9b128ef4=1463587952; _ga=GA1.2.2053141674.1463108258' -H 'Origin: http://pre3.papayamobile.com:1267' -H 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4' -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36' -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' -H 'Accept: application/json, text/javascript, */*; q=0.01' -H 'Referer: http://pre3.papayamobile.com:1267/shoptimize/campaign?type=website' -H 'X-Requested-With: XMLHttpRequest' -H 'Connection: keep-alive' --data 'product_type=0&account_id=2' --compressed""" pprint(parse_curl_str(s)) diff --git a/chrome/chrome.py b/chrome/chrome.py index f6c5b3f..6fa0769 100644 --- a/chrome/chrome.py +++ b/chrome/chrome.py @@ -22,7 +22,7 @@ def main(): url = "https://weread.qq.com/" url = "https://weread.qq.com/web/reader/b253292071697fe1b25cd24" - c = webdriver.Chrome() + c = webdriver.Chrome(executable_path='driver\chromedriver.exe') c.get(url) time.sleep(10) # 登陆一下 diff --git a/chrome/chrome_helper.py b/chrome/chrome_helper.py new file mode 100644 index 0000000..4dc6a69 --- /dev/null +++ b/chrome/chrome_helper.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import os +from log_handle import logger +import logging +import zipfile +import requests +import file_util + +# https://medium.com/drunk-wis/python-selenium-chrome-browser-%E8%88%87-driver-%E6%83%B1%E4%BA%BA%E7%9A%84%E7%89%88%E6%9C%AC%E7%AE%A1%E7%90%86-cbaf1d1861ce +CHROME_DRIVER_BASE_URL = "https://chromedriver.storage.googleapis.com" +CHROME_DRIVER_FOLDER = r".\chrome" +CHROME_DRIVER_MAPPING_FILE = r"{}\mapping.json".format(CHROME_DRIVER_FOLDER) +CHROME_DRIVER_EXE = r"{}\chromedriver.exe".format(CHROME_DRIVER_FOLDER) +CHROME_DRIVER_ZIP = r"{}\chromedriver_win32.zip".format(CHROME_DRIVER_FOLDER) + + +def init_dir(): + if not os.path.exists(CHROME_DRIVER_FOLDER): + os.makedirs(CHROME_DRIVER_FOLDER) + + +def get_chrome_driver_major_version(): + chrome_browser_path = r"C:\Program Files\Google\Chrome\Application\chrome.exe" + chrome_ver = file_util.get_file_version(chrome_browser_path) + logger.info(f'chrome version is {chrome_ver}') + chrome_major_ver = chrome_ver.split(".")[0] + logger.info(f'chrome driver major version is {chrome_major_ver}') + return chrome_major_ver + + +def get_latest_driver_version(browser_ver): + latest_api = "{}/LATEST_RELEASE_{}".format( + CHROME_DRIVER_BASE_URL, browser_ver) + logger.info(f'latest api is {latest_api}') + resp = requests.get(latest_api) + lastest_driver_version = resp.text.strip() + logger.info(f'latest driver version is {lastest_driver_version}') + return lastest_driver_version + + +def download_driver(driver_ver, dest_folder): + download_api = "{}/{}/chromedriver_win32.zip".format( + CHROME_DRIVER_BASE_URL, driver_ver) + logger.info(f'download api is {download_api}') + dest_path = os.path.join(dest_folder, os.path.basename(download_api)) + resp = requests.get(download_api, stream=True, timeout=300) + logger.info(f'download dest path is {dest_path}') + if resp.status_code == 200: + with open(dest_path, "wb") as f: + f.write(resp.content) + logger.info("Download driver completed") + else: + raise Exception("Download chrome driver failed") + + +def unzip_driver_to_target_path(src_file, dest_path): + with zipfile.ZipFile(src_file, 'r') as zip_ref: + zip_ref.extractall(dest_path) + logger.info("Unzip [{}] -> [{}]".format(src_file, dest_path)) + + +def read_driver_mapping_file(): + driver_mapping_dict = {} + if os.path.exists(CHROME_DRIVER_MAPPING_FILE): + driver_mapping_dict = file_util.read_json(CHROME_DRIVER_MAPPING_FILE) + logger.info(f'driver mapping dict is {driver_mapping_dict}') + return driver_mapping_dict + + +def check_browser_driver_available(): + chrome_major_ver = get_chrome_driver_major_version() + mapping_dict = read_driver_mapping_file() + driver_ver = get_latest_driver_version(chrome_major_ver) + + if chrome_major_ver not in mapping_dict: + download_driver(driver_ver, CHROME_DRIVER_FOLDER) + unzip_driver_to_target_path(CHROME_DRIVER_ZIP, CHROME_DRIVER_FOLDER) + + mapping_dict = { + chrome_major_ver: { + "driver_path": CHROME_DRIVER_EXE, + "driver_version": driver_ver + } + } + mapping_dict.update(mapping_dict) + file_util.write_json(CHROME_DRIVER_MAPPING_FILE, mapping_dict) + + +if __name__ == "__main__": + init_dir() + check_browser_driver_available() diff --git a/chrome/driver/chromedriver.exe b/chrome/driver/chromedriver.exe new file mode 100644 index 0000000..d51602d Binary files /dev/null and b/chrome/driver/chromedriver.exe differ diff --git a/chrome/file_util.py b/chrome/file_util.py new file mode 100644 index 0000000..204269e --- /dev/null +++ b/chrome/file_util.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import os +import json +from log_handle import logger + +from win32com import client as wincom_client + +""" +Environment Requirements: +Windows +1. Python 3+ +2. Google Chrome +Python Packages +pip install pypiwin32 +pip install requests +Folder Structure +--- chrome_helper.py + file_util.py + + +get_file_version() --- 取得版本號 +write_json() --- 寫 json file +read_json() --- 讀 json file +""" + + +def get_file_version(file_path): + logger.info('Get file version of [%s]', file_path) + if not os.path.isfile(file_path): + logger.error('{!r} is not found.'.format(file_path)) + raise FileNotFoundError('{!r} is not found.'.format(file_path)) + + wincom_obj = wincom_client.Dispatch('Scripting.FileSystemObject') + version = wincom_obj.GetFileVersion(file_path) + logger.info('The file version of [%s] is %s', file_path, version) + return version.strip() + + +def write_json(file_path, data): + with open(file_path, 'w', encoding='utf-8')as f: + json.dump(data, f, indent=2) + + +def read_json(file_path): + with open(file_path, encoding='utf-8')as f: + data = json.load(f) + return data diff --git a/chrome/log_handle.py b/chrome/log_handle.py new file mode 100644 index 0000000..0f3653e --- /dev/null +++ b/chrome/log_handle.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import logging +import logging.handlers +import os + +LOG_DIR = "log" +if not os.path.exists(LOG_DIR): + os.makedirs(LOG_DIR) +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) +file_handler = logging.handlers.TimedRotatingFileHandler('log/chrome.log', when='D', interval=1, backupCount=10) +file_handler.setLevel(logging.INFO) +file_handler.setFormatter( + logging.Formatter("%(asctime)s - %(levelname)s - %(filename)s[%(lineno)d] messages: %(message)s")) + +console_handler = logging.StreamHandler() +console_handler.setLevel(logging.INFO) +console_handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(filename)s[:%(lineno)d] - %(message)s")) + +logger.addHandler(file_handler) +logger.addHandler(console_handler) diff --git a/chrome/webdrive.py b/chrome/webdrive.py new file mode 100644 index 0000000..30661f4 --- /dev/null +++ b/chrome/webdrive.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from selenium import webdriver # 从selenium导入webdriver +import time + +""" +常用查找元素方法： + +find_element_by_id # ID +find_elements_by_class_name # class +find_elements_by_tag_name # 标签名 +find_elements_by_name # name +find_elements_by_link_text # a标签中的text查找（精确匹配） +find_elements_by_partial_link_text #a标签中的text查找（部分匹配即可） +find_elements_by_css_selector # css选择器查找 +find_elements_by_xpath # find_elements_by_xpath("//input")，请翻阅文档 +""" +driver = webdriver.Chrome('driver/chromedriver.exe') + +driver.get('https://www.baidu.com') + +input = driver.find_element_by_id('kw') +searchButton = driver.find_element_by_id('su') # 获取搜索按钮 +input.send_keys('python') +searchButton.click() +time.sleep(10) +driver.close() diff --git a/date_time/get_date_from_timestamp.py b/date_time/get_date_from_timestamp.py index 30d49f1..00638b1 100755 --- a/date_time/get_date_from_timestamp.py +++ b/date_time/get_date_from_timestamp.py @@ -57,7 +57,7 @@ def days_from_now(timestamp): s = 1456415452 print(datestr_from_stamp(s)) date = datetime_from_timestamp(s) - print(now-date).days + print((now-date).days) def main(): @@ -66,6 +66,7 @@ def main(): except: timestamp = time.time() print(datetime_from_timestamp(timestamp)) + days_from_now(timestamp) if __name__ == '__main__': diff --git a/date_time/get_day_of_month.py b/date_time/get_day_of_month.py index f9384cc..b22f4d1 100644 --- a/date_time/get_day_of_month.py +++ b/date_time/get_day_of_month.py @@ -19,5 +19,5 @@ date_first = datetime.datetime(year , month, 1) date_last = datetime.datetime(year, month+1, 1) - datetime.timedelta(1) -print date_first -print date_last +print(date_first) +print(date_last) diff --git a/date_time/month.py b/date_time/month.py index 2fc9285..9789ff4 100644 --- a/date_time/month.py +++ b/date_time/month.py @@ -49,5 +49,5 @@ def test(): if __name__ == '__main__': test() - import doctest + # import doctest # doctest.testmod() diff --git a/date_time/to_timestamp.py b/date_time/to_timestamp.py index 91a8726..a11354a 100755 --- a/date_time/to_timestamp.py +++ b/date_time/to_timestamp.py @@ -23,7 +23,7 @@ def parse(date_str): def test(): s = "2015-10-18 00:00:00" - print to_timestamp(s) + print(to_timestamp(s))