Dev

algorithm_datastructure/dict_tools.py

@@ -4,6 +4,7 @@
 
 class ObjectDict(dict):
     """from tornado.util.ObjectDict, access dict by dot"""
+
     def __getattr__(self, key):
         try:
             return self[key]
@@ -16,7 +17,6 @@ def __setattr__(self, key, value):
 
 import collections
 
-
 class TransformedDict(collections.MutableMapping):
     """A dictionary that applies an arbitrary key-altering
        function before accessing the keys"""
@@ -43,10 +43,12 @@ def __len__(self):
     def __keytransform__(self, key):
         return key
 
+
 # pip install fronzendict
 
 class FronzenDict(TransformedDict):
     """immutable dict, init by a dict"""
+
     def __init__(self, d):
         self.store = d
 
@@ -57,4 +59,7 @@ def __setitem__(self, key, value):
 
 # since python3.3, you can use immutable dict
 import types
-d_proxy = types.MappingProxyType(d)    # can add new key/value by assignment but can not assign key alrady exists
+
+d = {1: 'hello', 2: 'world'}
+d_proxy = types.MappingProxyType(d)  # can add new key/value by assignment but can not assign key alrady exists
+print(d_proxy)

autotest/curl_to_request.py

@@ -49,5 +49,5 @@ def parse_curl_str(s):
 
 
 if __name__ == '__main__':
-    s = """'curl http://pre3.papayamobile.com:1267/shoptimize/product/get_category_ids' -H 'Cookie: language=zh_CN; _gat=1; ppysid="PMs5bkeZejk5DZ9YuE+XRySRFW58DvWlgNYfI2YP+SQ="; save=true; email=fan_ll%40qq.com; password=6be037423107205d176a1d0d174402fac6b1dcc3d1b99b894fdbdf1d52ea9935; login_type=pmd; Hm_lvt_4f18dfc7029aaa9ff0b37ede9b128ef4=1463108257; Hm_lpvt_4f18dfc7029aaa9ff0b37ede9b128ef4=1463587952; _ga=GA1.2.2053141674.1463108258' -H 'Origin: http://pre3.papayamobile.com:1267' -H 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4' -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36' -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' -H 'Accept: application/json, text/javascript, */*; q=0.01' -H 'Referer: http://pre3.papayamobile.com:1267/shoptimize/campaign?type=website' -H 'X-Requested-With: XMLHttpRequest' -H 'Connection: keep-alive' --data 'product_type=0&account_id=2' --compressed"""
+    s = """curl 'http://pre3.papayamobile.com:1267/shoptimize/product/get_category_ids' -H 'Cookie: language=zh_CN; _gat=1; ppysid="PMs5bkeZejk5DZ9YuE+XRySRFW58DvWlgNYfI2YP+SQ="; save=true; email=fan_ll%40qq.com; password=6be037423107205d176a1d0d174402fac6b1dcc3d1b99b894fdbdf1d52ea9935; login_type=pmd; Hm_lvt_4f18dfc7029aaa9ff0b37ede9b128ef4=1463108257; Hm_lpvt_4f18dfc7029aaa9ff0b37ede9b128ef4=1463587952; _ga=GA1.2.2053141674.1463108258' -H 'Origin: http://pre3.papayamobile.com:1267' -H 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4' -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.94 Safari/537.36' -H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' -H 'Accept: application/json, text/javascript, */*; q=0.01' -H 'Referer: http://pre3.papayamobile.com:1267/shoptimize/campaign?type=website' -H 'X-Requested-With: XMLHttpRequest' -H 'Connection: keep-alive' --data 'product_type=0&account_id=2' --compressed"""
     pprint(parse_curl_str(s))

chrome/chrome.py

@@ -22,7 +22,7 @@ def main():
     url = "https://weread.qq.com/"
     url = "https://weread.qq.com/web/reader/b253292071697fe1b25cd24"
 
-    c = webdriver.Chrome()
+    c = webdriver.Chrome(executable_path='driver\chromedriver.exe')
     c.get(url)
     time.sleep(10)  # 登陆一下
 

chrome/chrome_helper.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import os
+from log_handle import logger
+import logging
+import zipfile
+import requests
+import file_util
+
+# https://medium.com/drunk-wis/python-selenium-chrome-browser-%E8%88%87-driver-%E6%83%B1%E4%BA%BA%E7%9A%84%E7%89%88%E6%9C%AC%E7%AE%A1%E7%90%86-cbaf1d1861ce
+CHROME_DRIVER_BASE_URL = "https://chromedriver.storage.googleapis.com"
+CHROME_DRIVER_FOLDER = r".\chrome"
+CHROME_DRIVER_MAPPING_FILE = r"{}\mapping.json".format(CHROME_DRIVER_FOLDER)
+CHROME_DRIVER_EXE = r"{}\chromedriver.exe".format(CHROME_DRIVER_FOLDER)
+CHROME_DRIVER_ZIP = r"{}\chromedriver_win32.zip".format(CHROME_DRIVER_FOLDER)
+
+
+def init_dir():
+    if not os.path.exists(CHROME_DRIVER_FOLDER):
+        os.makedirs(CHROME_DRIVER_FOLDER)
+
+
+def get_chrome_driver_major_version():
+    chrome_browser_path = r"C:\Program Files\Google\Chrome\Application\chrome.exe"
+    chrome_ver = file_util.get_file_version(chrome_browser_path)
+    logger.info(f'chrome version is {chrome_ver}')
+    chrome_major_ver = chrome_ver.split(".")[0]
+    logger.info(f'chrome driver major version is {chrome_major_ver}')
+    return chrome_major_ver
+
+
+def get_latest_driver_version(browser_ver):
+    latest_api = "{}/LATEST_RELEASE_{}".format(
+        CHROME_DRIVER_BASE_URL, browser_ver)
+    logger.info(f'latest api is {latest_api}')
+    resp = requests.get(latest_api)
+    lastest_driver_version = resp.text.strip()
+    logger.info(f'latest driver version is {lastest_driver_version}')
+    return lastest_driver_version
+
+
+def download_driver(driver_ver, dest_folder):
+    download_api = "{}/{}/chromedriver_win32.zip".format(
+        CHROME_DRIVER_BASE_URL, driver_ver)
+    logger.info(f'download api is {download_api}')
+    dest_path = os.path.join(dest_folder, os.path.basename(download_api))
+    resp = requests.get(download_api, stream=True, timeout=300)
+    logger.info(f'download dest path is {dest_path}')
+    if resp.status_code == 200:
+        with open(dest_path, "wb") as f:
+            f.write(resp.content)
+        logger.info("Download driver completed")
+    else:
+        raise Exception("Download chrome driver failed")
+
+
+def unzip_driver_to_target_path(src_file, dest_path):
+    with zipfile.ZipFile(src_file, 'r') as zip_ref:
+        zip_ref.extractall(dest_path)
+    logger.info("Unzip [{}] -> [{}]".format(src_file, dest_path))
+
+
+def read_driver_mapping_file():
+    driver_mapping_dict = {}
+    if os.path.exists(CHROME_DRIVER_MAPPING_FILE):
+        driver_mapping_dict = file_util.read_json(CHROME_DRIVER_MAPPING_FILE)
+    logger.info(f'driver mapping dict is {driver_mapping_dict}')
+    return driver_mapping_dict
+
+
+def check_browser_driver_available():
+    chrome_major_ver = get_chrome_driver_major_version()
+    mapping_dict = read_driver_mapping_file()
+    driver_ver = get_latest_driver_version(chrome_major_ver)
+
+    if chrome_major_ver not in mapping_dict:
+        download_driver(driver_ver, CHROME_DRIVER_FOLDER)
+        unzip_driver_to_target_path(CHROME_DRIVER_ZIP, CHROME_DRIVER_FOLDER)
+
+        mapping_dict = {
+            chrome_major_ver: {
+                "driver_path": CHROME_DRIVER_EXE,
+                "driver_version": driver_ver
+            }
+        }
+        mapping_dict.update(mapping_dict)
+        file_util.write_json(CHROME_DRIVER_MAPPING_FILE, mapping_dict)
+
+
+if __name__ == "__main__":
+    init_dir()
+    check_browser_driver_available()

chrome/file_util.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import os
+import json
+from log_handle import logger
+
+from win32com import client as wincom_client
+
+"""
+Environment Requirements:
+Windows
+1. Python 3+
+2. Google Chrome
+Python Packages
+pip install pypiwin32
+pip install requests
+Folder Structure
+--- chrome_helper.py
+    file_util.py   
+
+
+get_file_version() --- 取得版本號
+write_json() --- 寫 json file
+read_json() --- 讀 json file
+"""
+
+
+def get_file_version(file_path):
+    logger.info('Get file version of [%s]', file_path)
+    if not os.path.isfile(file_path):
+        logger.error('{!r} is not found.'.format(file_path))
+        raise FileNotFoundError('{!r} is not found.'.format(file_path))
+
+    wincom_obj = wincom_client.Dispatch('Scripting.FileSystemObject')
+    version = wincom_obj.GetFileVersion(file_path)
+    logger.info('The file version of [%s] is %s', file_path, version)
+    return version.strip()
+
+
+def write_json(file_path, data):
+    with  open(file_path, 'w', encoding='utf-8')as f:
+        json.dump(data, f, indent=2)
+
+
+def read_json(file_path):
+    with open(file_path, encoding='utf-8')as f:
+        data = json.load(f)
+    return data

chrome/log_handle.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import logging
+import logging.handlers
+import os
+
+LOG_DIR = "log"
+if not os.path.exists(LOG_DIR):
+    os.makedirs(LOG_DIR)
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+file_handler = logging.handlers.TimedRotatingFileHandler('log/chrome.log', when='D', interval=1, backupCount=10)
+file_handler.setLevel(logging.INFO)
+file_handler.setFormatter(
+    logging.Formatter("%(asctime)s - %(levelname)s - %(filename)s[%(lineno)d] messages: %(message)s"))
+
+console_handler = logging.StreamHandler()
+console_handler.setLevel(logging.INFO)
+console_handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(filename)s[:%(lineno)d] - %(message)s"))
+
+logger.addHandler(file_handler)
+logger.addHandler(console_handler)

chrome/webdrive.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from selenium import webdriver  # 从selenium导入webdriver
+import time
+
+"""
+常用查找元素方法：
+
+find_element_by_id # ID
+find_elements_by_class_name # class
+find_elements_by_tag_name # 标签名
+find_elements_by_name # name
+find_elements_by_link_text # a标签中的text查找（精确匹配）
+find_elements_by_partial_link_text #a标签中的text查找（部分匹配即可）
+find_elements_by_css_selector # css选择器查找
+find_elements_by_xpath # find_elements_by_xpath("//input")，请翻阅文档
+"""
+driver = webdriver.Chrome('driver/chromedriver.exe')
+
+driver.get('https://www.baidu.com')
+
+input = driver.find_element_by_id('kw')
+searchButton = driver.find_element_by_id('su')  # 获取搜索按钮
+input.send_keys('python')
+searchButton.click()
+time.sleep(10)
+driver.close()

date_time/get_date_from_timestamp.py

@@ -57,7 +57,7 @@ def days_from_now(timestamp):
     s = 1456415452
     print(datestr_from_stamp(s))
     date = datetime_from_timestamp(s)
-    print(now-date).days
+    print((now-date).days)
 
 
 def main():
@@ -66,6 +66,7 @@ def main():
     except:
         timestamp = time.time()
     print(datetime_from_timestamp(timestamp))
+    days_from_now(timestamp)
 
 
 if __name__ == '__main__':

date_time/get_day_of_month.py

@@ -19,5 +19,5 @@
 date_first = datetime.datetime(year , month, 1)
 date_last = datetime.datetime(year, month+1, 1) - datetime.timedelta(1)
 
-print date_first
-print date_last
+print(date_first)
+print(date_last)

date_time/month.py

@@ -49,5 +49,5 @@ def test():
 
 if __name__ == '__main__':
     test()
-    import doctest
+    # import doctest
     # doctest.testmod()

date_time/to_timestamp.py

@@ -23,7 +23,7 @@ def parse(date_str):
 
 def test():
     s = "2015-10-18 00:00:00"
-    print to_timestamp(s)
+    print(to_timestamp(s))