LINT: Adding scripts directory to lint, and fixing flake issues on them (#18949) (#19344)

Author: datapythonista

ci/lint.sh

@@ -30,6 +30,13 @@ if [ "$LINT" ]; then
     fi
     echo "Linting asv_bench/benchmarks/*.py DONE"
 
+    echo "Linting scripts/*.py"
+    flake8 scripts --filename=*.py
+    if [ $? -ne "0" ]; then
+        RET=1
+    fi
+    echo "Linting scripts/*.py DONE"
+
     echo "Linting *.pyx"
     flake8 pandas --filename=*.pyx --select=E501,E302,E203,E111,E114,E221,E303,E128,E231,E126,E265,E305,E301,E127,E261,E271,E129,W291,E222,E241,E123,F403
     if [ $? -ne "0" ]; then

scripts/announce.py

@@ -17,9 +17,11 @@
     $ PYTHONPATH=.. ./api_rst_coverage.py
 
 """
-import pandas as pd
-import inspect
+import os
 import re
+import inspect
+import pandas as pd
+
 
 def main():
     # classes whose members to check
@@ -61,21 +63,26 @@ def add_notes(x):
     # class members
     class_members = set()
     for cls in classes:
-        class_members.update([cls.__name__ + '.' + x[0] for x in inspect.getmembers(cls)])
+        for member in inspect.getmembers(cls):
+            class_members.add('{cls}.{member}'.format(cls=cls.__name__,
+                                                      member=member[0]))
 
     # class members referenced in api.rst
     api_rst_members = set()
-    file_name = '../doc/source/api.rst'
-    with open(file_name, 'r') as f:
-        pattern = re.compile('({})\.(\w+)'.format('|'.join(cls.__name__ for cls in classes)))
+    base_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    api_rst_fname = os.path.join(base_path, 'doc', 'source', 'api.rst')
+    class_names = (cls.__name__ for cls in classes)
+    pattern = re.compile('({})\.(\w+)'.format('|'.join(class_names)))
+    with open(api_rst_fname, 'r') as f:
         for line in f:
             match = pattern.search(line)
             if match:
                 api_rst_members.add(match.group(0))
 
     print()
     print("Documented members in api.rst that aren't actual class members:")
-    for x in sorted(api_rst_members.difference(class_members), key=class_name_sort_key):
+    for x in sorted(api_rst_members.difference(class_members),
+                    key=class_name_sort_key):
         print(x)
 
     print()
@@ -86,5 +93,6 @@ def add_notes(x):
         if '._' not in x:
             print(add_notes(x))
 
+
 if __name__ == "__main__":
     main()

scripts/api_rst_coverage.py

@@ -1,135 +1,148 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-
 # copyright 2013, y-p @ github
-
-from __future__ import print_function
-from pandas.compat import range, lrange, map, string_types, text_type
-
-"""Search the git history for all commits touching a named method
+"""
+Search the git history for all commits touching a named method
 
 You need the sh module to run this
-WARNING: this script uses git clean -f, running it on a repo with untracked files
-will probably erase them.
+WARNING: this script uses git clean -f, running it on a repo with untracked
+files will probably erase them.
+
+Usage::
+    $ ./find_commits_touching_func.py  (see arguments below)
 """
+from __future__ import print_function
 import logging
 import re
 import os
+import argparse
 from collections import namedtuple
-from pandas.compat import parse_date
-
+from pandas.compat import lrange, map, string_types, text_type, parse_date
 try:
     import sh
 except ImportError:
-    raise ImportError("The 'sh' package is required in order to run this script. ")
+    raise ImportError("The 'sh' package is required to run this script.")
 
-import argparse
 
 desc = """
 Find all commits touching a specified function across the codebase.
 """.strip()
 argparser = argparse.ArgumentParser(description=desc)
 argparser.add_argument('funcname', metavar='FUNCNAME',
-                   help='Name of function/method to search for changes on.')
+                       help='Name of function/method to search for changes on')
 argparser.add_argument('-f', '--file-masks', metavar='f_re(,f_re)*',
                        default=["\.py.?$"],
-                   help='comma separated list of regexes to match filenames against\n'+
-                   'defaults all .py? files')
+                       help='comma separated list of regexes to match '
+                       'filenames against\ndefaults all .py? files')
 argparser.add_argument('-d', '--dir-masks', metavar='d_re(,d_re)*',
                        default=[],
-                   help='comma separated list of regexes to match base path against')
+                       help='comma separated list of regexes to match base '
+                       'path against')
 argparser.add_argument('-p', '--path-masks', metavar='p_re(,p_re)*',
                        default=[],
-                   help='comma separated list of regexes to match full file path against')
+                       help='comma separated list of regexes to match full '
+                       'file path against')
 argparser.add_argument('-y', '--saw-the-warning',
-                       action='store_true',default=False,
-                   help='must specify this to run, acknowledge you realize this will erase untracked files')
+                       action='store_true', default=False,
+                       help='must specify this to run, acknowledge you '
+                       'realize this will erase untracked files')
 argparser.add_argument('--debug-level',
                        default="CRITICAL",
-                   help='debug level of messages (DEBUG,INFO,etc...)')
-
+                       help='debug level of messages (DEBUG, INFO, etc...)')
 args = argparser.parse_args()
 
 
 lfmt = logging.Formatter(fmt='%(levelname)-8s %(message)s',
-    datefmt='%m-%d %H:%M:%S'
-)
-
+                         datefmt='%m-%d %H:%M:%S')
 shh = logging.StreamHandler()
 shh.setFormatter(lfmt)
-
-logger=logging.getLogger("findit")
+logger = logging.getLogger("findit")
 logger.addHandler(shh)
 
+Hit = namedtuple("Hit", "commit path")
+HASH_LEN = 8
 
-Hit=namedtuple("Hit","commit path")
-HASH_LEN=8
 
 def clean_checkout(comm):
-    h,s,d = get_commit_vitals(comm)
+    h, s, d = get_commit_vitals(comm)
     if len(s) > 60:
         s = s[:60] + "..."
-    s=s.split("\n")[0]
-    logger.info("CO: %s %s" % (comm,s ))
+    s = s.split("\n")[0]
+    logger.info("CO: %s %s" % (comm, s))
 
-    sh.git('checkout', comm ,_tty_out=False)
+    sh.git('checkout', comm, _tty_out=False)
     sh.git('clean', '-f')
 
-def get_hits(defname,files=()):
-    cs=set()
+
+def get_hits(defname, files=()):
+    cs = set()
     for f in files:
         try:
-            r=sh.git('blame', '-L', '/def\s*{start}/,/def/'.format(start=defname),f,_tty_out=False)
+            r = sh.git('blame',
+                       '-L',
+                       '/def\s*{start}/,/def/'.format(start=defname),
+                       f,
+                       _tty_out=False)
         except sh.ErrorReturnCode_128:
             logger.debug("no matches in %s" % f)
             continue
 
         lines = r.strip().splitlines()[:-1]
         # remove comment lines
-        lines = [x for x in lines if not re.search("^\w+\s*\(.+\)\s*#",x)]
-        hits = set(map(lambda x: x.split(" ")[0],lines))
-        cs.update(set(Hit(commit=c,path=f) for c in hits))
+        lines = [x for x in lines if not re.search("^\w+\s*\(.+\)\s*#", x)]
+        hits = set(map(lambda x: x.split(" ")[0], lines))
+        cs.update(set(Hit(commit=c, path=f) for c in hits))
 
     return cs
 
-def get_commit_info(c,fmt,sep='\t'):
-    r=sh.git('log', "--format={}".format(fmt), '{}^..{}'.format(c,c),"-n","1",_tty_out=False)
+
+def get_commit_info(c, fmt, sep='\t'):
+    r = sh.git('log',
+               "--format={}".format(fmt),
+               '{}^..{}'.format(c, c),
+               "-n",
+               "1",
+               _tty_out=False)
     return text_type(r).split(sep)
 
-def get_commit_vitals(c,hlen=HASH_LEN):
-    h,s,d= get_commit_info(c,'%H\t%s\t%ci',"\t")
-    return h[:hlen],s,parse_date(d)
 
-def file_filter(state,dirname,fnames):
-    if args.dir_masks and not any(re.search(x,dirname) for x in args.dir_masks):
+def get_commit_vitals(c, hlen=HASH_LEN):
+    h, s, d = get_commit_info(c, '%H\t%s\t%ci', "\t")
+    return h[:hlen], s, parse_date(d)
+
+
+def file_filter(state, dirname, fnames):
+    if (args.dir_masks and
+            not any(re.search(x, dirname) for x in args.dir_masks)):
         return
     for f in fnames:
-        p = os.path.abspath(os.path.join(os.path.realpath(dirname),f))
-        if  any(re.search(x,f) for x in args.file_masks)\
-            or any(re.search(x,p) for x in args.path_masks):
+        p = os.path.abspath(os.path.join(os.path.realpath(dirname), f))
+        if (any(re.search(x, f) for x in args.file_masks) or
+                any(re.search(x, p) for x in args.path_masks)):
             if os.path.isfile(p):
                 state['files'].append(p)
 
-def search(defname,head_commit="HEAD"):
-    HEAD,s = get_commit_vitals("HEAD")[:2]
-    logger.info("HEAD at %s: %s" % (HEAD,s))
+
+def search(defname, head_commit="HEAD"):
+    HEAD, s = get_commit_vitals("HEAD")[:2]
+    logger.info("HEAD at %s: %s" % (HEAD, s))
     done_commits = set()
     # allhits = set()
     files = []
     state = dict(files=files)
-    os.path.walk('.',file_filter,state)
+    os.walk('.', file_filter, state)
     # files now holds a list of paths to files
 
     # seed with hits from q
-    allhits= set(get_hits(defname, files = files))
+    allhits = set(get_hits(defname, files=files))
     q = set([HEAD])
     try:
         while q:
-            h=q.pop()
+            h = q.pop()
             clean_checkout(h)
-            hits = get_hits(defname, files = files)
+            hits = get_hits(defname, files=files)
             for x in hits:
-                prevc = get_commit_vitals(x.commit+"^")[0]
+                prevc = get_commit_vitals(x.commit + "^")[0]
                 if prevc not in done_commits:
                     q.add(prevc)
             allhits.update(hits)
@@ -141,43 +154,46 @@ def search(defname,head_commit="HEAD"):
         clean_checkout(HEAD)
     return allhits
 
+
 def pprint_hits(hits):
-    SUBJ_LEN=50
+    SUBJ_LEN = 50
     PATH_LEN = 20
-    hits=list(hits)
+    hits = list(hits)
     max_p = 0
     for hit in hits:
-        p=hit.path.split(os.path.realpath(os.curdir)+os.path.sep)[-1]
-        max_p=max(max_p,len(p))
+        p = hit.path.split(os.path.realpath(os.curdir) + os.path.sep)[-1]
+        max_p = max(max_p, len(p))
 
     if max_p < PATH_LEN:
         SUBJ_LEN += PATH_LEN - max_p
         PATH_LEN = max_p
 
     def sorter(i):
-        h,s,d=get_commit_vitals(hits[i].commit)
-        return hits[i].path,d
+        h, s, d = get_commit_vitals(hits[i].commit)
+        return hits[i].path, d
 
-    print("\nThese commits touched the %s method in these files on these dates:\n" \
-          % args.funcname)
-    for i in sorted(lrange(len(hits)),key=sorter):
+    print(('\nThese commits touched the %s method in these files '
+           'on these dates:\n') % args.funcname)
+    for i in sorted(lrange(len(hits)), key=sorter):
         hit = hits[i]
-        h,s,d=get_commit_vitals(hit.commit)
-        p=hit.path.split(os.path.realpath(os.curdir)+os.path.sep)[-1]
+        h, s, d = get_commit_vitals(hit.commit)
+        p = hit.path.split(os.path.realpath(os.curdir) + os.path.sep)[-1]
 
         fmt = "{:%d} {:10} {:<%d} {:<%d}" % (HASH_LEN, SUBJ_LEN, PATH_LEN)
         if len(s) > SUBJ_LEN:
-            s = s[:SUBJ_LEN-5] + " ..."
-        print(fmt.format(h[:HASH_LEN],d.isoformat()[:10],s,p[-20:]) )
+            s = s[:SUBJ_LEN - 5] + " ..."
+        print(fmt.format(h[:HASH_LEN], d.isoformat()[:10], s, p[-20:]))
 
     print("\n")
 
+
 def main():
     if not args.saw_the_warning:
         argparser.print_help()
         print("""
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-WARNING: this script uses git clean -f, running it on a repo with untracked files.
+WARNING:
+this script uses git clean -f, running it on a repo with untracked files.
 It's recommended that you make a fresh clone and run from its root directory.
 You must specify the -y argument to ignore this warning.
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -190,12 +206,11 @@ def main():
     if isinstance(args.dir_masks, string_types):
         args.dir_masks = args.dir_masks.split(',')
 
-    logger.setLevel(getattr(logging,args.debug_level))
+    logger.setLevel(getattr(logging, args.debug_level))
 
-    hits=search(args.funcname)
+    hits = search(args.funcname)
     pprint_hits(hits)
 
-    pass
 
 if __name__ == "__main__":
     import sys