Format and check all Python code using python black

.gitignore

@@ -6,6 +6,7 @@
 *.pyc
 *.swp
 *~
+.venv
 .DS_Store
 .rebar/
 .eunit/

.travis.yml

@@ -11,17 +11,19 @@ otp_release:
 
 addons:
   apt:
+    sources:
+      - deadsnakes
     packages:
     - build-essential
     - curl
     - libcurl4-openssl-dev
     - libicu-dev
     - libmozjs185-dev
     - pkg-config
-    - python3
+    - python3.6
+    - python3.6-venv
     - python3-requests
     - python3-sphinx
-    - python3.4-venv
 #    - sphinx-rtd-theme
     - help2man
     - shunit2
@@ -50,6 +52,8 @@ before_script:
   - rm -rf /tmp/couchjslogs
   - mkdir -p /tmp/couchjslogs
   - ./configure -c --disable-docs --disable-fauxton
+  - python3.6 -m venv /tmp/.venv
+  - source /tmp/.venv/bin/activate
 
 script:
    - make check

Makefile

@@ -140,6 +140,7 @@ check: all
 	@$(MAKE) test-cluster-without-quorum
 	@$(MAKE) eunit
 	@$(MAKE) javascript
+	@$(MAKE) python-black
 	@$(MAKE) mango-test
 #	@$(MAKE) build-test
 
@@ -171,6 +172,27 @@ soak-eunit: couch
 	@$(REBAR) setup_eunit 2> /dev/null
 	while [ $$? -eq 0 ] ; do $(REBAR) -r eunit $(EUNIT_OPTS) ; done
 
+.venv/bin/black:
+	@python3 -m venv .venv
+	@.venv/bin/pip3 install black || touch .venv/bin/black
+
+# Python code formatter - only runs if we're on Python 3.6 or greater
+python-black: .venv/bin/black
+	@python3 -c "import sys; exit(1 if sys.version_info < (3,6) else 0)" || \
+	       echo "Python formatter not supported on Python < 3.6; check results on a newer platform"
+	@python3 -c "import sys; exit(1 if sys.version_info >= (3,6) else 0)" || \
+		LC_ALL=C.UTF-8 LANG=C.UTF-8 .venv/bin/black --check \
+		--exclude="build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|src/rebar/pr2relnotes.py|src/fauxton" \
+		. dev/run rel/overlay/bin/couchup test/javascript/run
+
+python-black-update: .venv/bin/black
+	@python3 -c "import sys; exit(1 if sys.version_info < (3,6) else 0)" || \
+	       echo "Python formatter not supported on Python < 3.6; check results on a newer platform"
+	@python3 -c "import sys; exit(1 if sys.version_info >= (3,6) else 0)" || \
+		LC_ALL=C.UTF-8 LANG=C.UTF-8 .venv/bin/black \
+		--exclude="build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|src/rebar/pr2relnotes.py|src/fauxton" \
+		. dev/run rel/overlay/bin/couchup test/javascript/run
+
 .PHONY: elixir
 elixir: elixir-check-formatted devclean
 	@dev/run -a adm:pass --no-eval test/elixir/run
@@ -284,9 +306,9 @@ build-test:
 # target: mango-test - Run Mango tests
 mango-test: devclean all
 	@cd src/mango && \
-		python3 -m venv venv && \
-		venv/bin/pip3 install -r requirements.txt
-	@cd src/mango && ../../dev/run -n 1 --admin=testuser:testpass venv/bin/nosetests
+		python3 -m venv .venv && \
+		.venv/bin/pip3 install -r requirements.txt
+	@cd src/mango && ../../dev/run -n 1 --admin=testuser:testpass .venv/bin/nosetests
 
 ################################################################################
 # Developing
@@ -399,6 +421,7 @@ clean:
 	@rm -rf src/couch/priv/{couchspawnkillable,couchjs}
 	@rm -rf share/server/main.js share/server/main-coffee.js
 	@rm -rf tmp dev/data dev/lib dev/logs
+	@rm -rf src/mango/.venv
 	@rm -f src/couch/priv/couchspawnkillable
 	@rm -f src/couch/priv/couch_js/config.h
 	@rm -f dev/boot_node.beam dev/pbkdf2.pyc log/crash.log

Makefile.win

@@ -119,6 +119,7 @@ check: all
 	@$(MAKE) -f Makefile.win test-cluster-without-quorum
 	@$(MAKE) -f Makefile.win eunit
 	@$(MAKE) -f Makefile.win javascript
+	@$(MAKE) -f Makefile.win python-black
 	@$(MAKE) -f Makefile.win mango-test
 #	@$(MAKE) -f Makefile.win elixir
 
@@ -142,6 +143,26 @@ just-eunit: export ERL_AFLAGS = $(shell echo "-config rel/files/eunit.config")
 just-eunit:
 	@$(REBAR) -r eunit $(EUNIT_OPTS)
 
+.venv/bin/black:
+	@python.exe -m venv .venv
+	@.venv\Scripts\pip3.exe install black || copy /b .venv\Scripts\black.exe +,,
+
+# Python code formatter - only runs if we're on Python 3.6 or greater
+python-black: .venv/bin/black
+	@python.exe -c "import sys; exit(1 if sys.version_info < (3,6) else 0)" || \
+		echo "Python formatter not supported on Python < 3.6; check results on a newer platform"
+	@python.exe -c "import sys; exit(1 if sys.version_info >= (3,6) else 0)" || \
+		.venv\Scripts\black.exe --check \
+		--exclude="build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|src/rebar/pr2relnotes.py|src/fauxton" \
+		. dev\run rel\overlay\bin\couchup test\javascript\run
+
+python-black-update: .venv/bin/black
+	@python.exe -c "import sys; exit(1 if sys.version_info < (3,6) else 0)" || \
+		echo "Python formatter not supported on Python < 3.6; check results on a newer platform"
+	@python.exe -c "import sys; exit(1 if sys.version_info >= (3,6) else 0)" || \
+		.venv\Scripts\black.exe \
+		--exclude="build/|buck-out/|dist/|_build/|\.git/|\.hg/|\.mypy_cache/|\.nox/|\.tox/|\.venv/|src/rebar/pr2relnotes.py|src/fauxton" \
+		. dev\run rel\overlay\bin\couchup test\javascript\run
 
 .PHONY: elixir
 elixir: elixir-check-formatted devclean
@@ -204,9 +225,9 @@ endif
 .PHONY: mango-test
 mango-test: devclean all
 	@cd src\mango && \
-		python.exe -m venv venv && \
-		venv\Scripts\pip.exe install -r requirements.txt
-	@cd src\mango && venv\Scripts\python.exe ..\..\dev\run -n 1 --admin=testuser:testpass venv\Scripts\nosetests
+		python.exe -m venv .venv && \
+		.venv\Scripts\pip.exe install -r requirements.txt
+	@cd src\mango && .venv\Scripts\python.exe ..\..\dev\run -n 1 --admin=testuser:testpass .venv\Scripts\nosetests
 
 
 .PHONY: check-qs
@@ -329,6 +350,7 @@ clean:
 	-@rmdir /s/q dev\data
 	-@rmdir /s/q dev\lib
 	-@rmdir /s/q dev\logs
+	-@rmdir /s/q src\mango\.venv
 	-@del /f/q src\couch\priv\couch_js\config.h
 	-@del /f/q dev\boot_node.beam dev\pbkdf2.pyc log\crash.log
 

build-aux/logfile-uploader.py

@@ -13,7 +13,6 @@
 # the License.
 
 
-
 import datetime
 import glob
 import json
@@ -26,96 +25,111 @@
 COUCH_URL = "https://couchdb-vm2.apache.org/ci_errorlogs"
 TARFILE = "couchlog.tar.gz"
 
+
 def _tojson(req):
     """Support requests v0.x as well as 1.x+"""
-    if requests.__version__[0] == '0':
+    if requests.__version__[0] == "0":
         return json.loads(req.content)
     return req.json()
 
+
 def collect_logfiles():
     """ Find and tarball all logfiles """
-    tb = tarfile.open(name=TARFILE, mode='w:gz')
+    tb = tarfile.open(name=TARFILE, mode="w:gz")
     # EUnit
-    for log in glob.glob('src/*/.eunit/couch.log'):
+    for log in glob.glob("src/*/.eunit/couch.log"):
         tb.add(log)
     # JS harness
-    for log in glob.glob('dev/logs/node1.log'):
+    for log in glob.glob("dev/logs/node1.log"):
         tb.add(log)
     # couchjs OS process IO logs
-    for log in glob.glob('/tmp/couchjslogs/*'):
+    for log in glob.glob("/tmp/couchjslogs/*"):
         tb.add(log)
     tb.close()
 
+
 def build_ci_doc():
     """ Build a metadata document with relevant detail from CI env """
     doc = {}
-    if 'TRAVIS' in os.environ:
-        doc['builder'] = 'travis'
-        doc['build_id'] = os.environ['TRAVIS_JOB_ID']
-        doc['erlang'] = os.environ['TRAVIS_OTP_RELEASE']
-        doc['url'] = 'https://travis-ci.org/apache/couchdb/jobs/' + \
-            os.environ['TRAVIS_JOB_ID']
-        doc['branch'] = os.environ['TRAVIS_BRANCH']
-        doc['commit'] = os.environ['TRAVIS_COMMIT']
-        doc['repo'] = 'https://github.com/' + os.environ['TRAVIS_REPO_SLUG']
-    elif 'JENKINS_URL' in os.environ:
-        doc['builder'] = 'jenkins'
-        doc['build_id'] = os.environ['BUILD_NUMBER']
-        doc['url'] = os.environ['BUILD_URL']
-        doc['branch'] = os.environ['BRANCH_NAME']
-        doc['repo'] = 'https://github.com/apache/couchdb'
+    if "TRAVIS" in os.environ:
+        doc["builder"] = "travis"
+        doc["build_id"] = os.environ["TRAVIS_JOB_ID"]
+        doc["erlang"] = os.environ["TRAVIS_OTP_RELEASE"]
+        doc["url"] = (
+            "https://travis-ci.org/apache/couchdb/jobs/" + os.environ["TRAVIS_JOB_ID"]
+        )
+        doc["branch"] = os.environ["TRAVIS_BRANCH"]
+        doc["commit"] = os.environ["TRAVIS_COMMIT"]
+        doc["repo"] = "https://github.com/" + os.environ["TRAVIS_REPO_SLUG"]
+    elif "JENKINS_URL" in os.environ:
+        doc["builder"] = "jenkins"
+        doc["build_id"] = os.environ["BUILD_NUMBER"]
+        doc["url"] = os.environ["BUILD_URL"]
+        doc["branch"] = os.environ["BRANCH_NAME"]
+        doc["repo"] = "https://github.com/apache/couchdb"
     else:
-        doc['builder'] = 'manual'
+        doc["builder"] = "manual"
         # TODO: shell out to get correct repo, commit, branch info?
-        doc['repo'] = 'https://github.com/apache/couchdb'
-        doc['build_id'] = str(time.time())
+        doc["repo"] = "https://github.com/apache/couchdb"
+        doc["build_id"] = str(time.time())
 
     # shorten doc id
-    repo = doc['repo'].split('/')[-1]
-    repo = repo.replace('.git', '')
-
-    doc['_id'] = doc['builder'] + '-' + repo + '-' + \
-        doc['build_id'] + \
-        '-' + datetime.datetime.utcnow().isoformat()
+    repo = doc["repo"].split("/")[-1]
+    repo = repo.replace(".git", "")
+
+    doc["_id"] = (
+        doc["builder"]
+        + "-"
+        + repo
+        + "-"
+        + doc["build_id"]
+        + "-"
+        + datetime.datetime.utcnow().isoformat()
+    )
 
     return doc
 
+
 def upload_logs():
     try:
-        lp = os.environ['COUCHAUTH'].split(':')
+        lp = os.environ["COUCHAUTH"].split(":")
     except KeyError as e:
-        print ("ERROR: COUCHAUTH credentials unavailable! "
-            "Unable to upload logfiles.")
+        print("ERROR: COUCHAUTH credentials unavailable! " "Unable to upload logfiles.")
         exit(1)
 
     creds = (lp[0], lp[1])
     doc = build_ci_doc()
-    req = requests.post(COUCH_URL,
+    req = requests.post(
+        COUCH_URL,
         data=json.dumps(doc),
         auth=creds,
-        headers={'Content-type': 'application/json'})
+        headers={"Content-type": "application/json"},
+    )
     req.raise_for_status()
     req = _tojson(req)
-    with open(TARFILE, 'rb') as f:
+    with open(TARFILE, "rb") as f:
         # ancient versions of requests break if data is iterable
         fdata = f.read()
-        req2 = requests.put(COUCH_URL + '/' + doc['_id'] + '/' + TARFILE,
-            headers={'Content-type': 'application/x-gtar'},
+        req2 = requests.put(
+            COUCH_URL + "/" + doc["_id"] + "/" + TARFILE,
+            headers={"Content-type": "application/x-gtar"},
             auth=creds,
-            params={'rev': req['rev']},
-            data=fdata)
+            params={"rev": req["rev"]},
+            data=fdata,
+        )
     req2.raise_for_status()
     return req2
 
 
 def main():
     """ Find latest logfile and upload to Couch logfile db. """
-    print ("Uploading logfiles...")
+    print("Uploading logfiles...")
     collect_logfiles()
     req = upload_logs()
-    print (req.url.split('?')[0])
-    print (req.content)
-    print ("Upload complete!")
+    print(req.url.split("?")[0])
+    print(req.content)
+    print("Upload complete!")
+
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()