renamed corpora to corpus, added deprecation warning

.gitignore

@@ -2,6 +2,7 @@
 __pycache__/
 *.py[cod]
 *$py.class
+.pytest_cache
 
 # C extensions
 *.so

quantgov/__init__.py

@@ -3,11 +3,15 @@
 
 __all__ = [
     'corpora',
+    'corpus',
     'estimator',
     'project',
     'utils',
 ]
 
-from .corpora.utils import load_driver
+
+from . import corpora  # Backwards compatibility
+
+from .utils import load_driver
 
 __version__ = '0.4.0.dev'

quantgov/__main__.py

@@ -15,7 +15,7 @@
 import requests
 
 import quantgov
-import quantgov.corpora.builtins
+import quantgov.corpus.builtins
 
 from pathlib import Path
 
@@ -40,7 +40,7 @@ def parse_args():
     # Corpus command
     corpus = subparsers.add_parser('corpus')
     corpus_subcommands = corpus.add_subparsers(dest='subcommand')
-    for command, builtin in quantgov.corpora.builtins.commands.items():
+    for command, builtin in quantgov.corpus.builtins.commands.items():
         subcommand = corpus_subcommands.add_parser(
             command, help=builtin.cli.help)
         subcommand.add_argument(
@@ -161,7 +161,7 @@ def start_component(args):
 def run_corpus_builtin(args):
     driver = quantgov.load_driver(args.corpus)
     writer = csv.writer(args.outfile)
-    builtin = quantgov.corpora.builtins.commands[args.subcommand]
+    builtin = quantgov.corpus.builtins.commands[args.subcommand]
     func_args = {i: j for i, j in vars(args).items()
                  if i not in {'command', 'subcommand', 'outfile', 'corpus'}}
     writer.writerow(driver.index_labels + builtin.get_columns(func_args))

quantgov/corpora/__init__.py

@@ -1,4 +1,6 @@
-from .structures import (
+import warnings
+
+from ..corpus import (
     Document,
     CorpusStreamer,
     CorpusDriver,
@@ -7,3 +9,8 @@
     NamePatternCorpusDriver,
     IndexDriver
 )
+
+warnings.warn(
+    ("quantgov.corpora has been moved to quantgov.corpus and will be removed"
+     " in a future version."),
+    DeprecationWarning)

quantgov/corpus/__init__.py

@@ -0,0 +1,9 @@
+from .structures import (
+    Document,
+    CorpusStreamer,
+    CorpusDriver,
+    FlatFileCorpusDriver,
+    RecursiveDirectoryCorpusDriver,
+    NamePatternCorpusDriver,
+    IndexDriver
+)

quantgov/utils.py

@@ -1,7 +1,22 @@
 # TODO: Docstrings
+
 import collections
 import concurrent.futures
-import multiprocessing
+import os
+import sys
+
+from pathlib import Path
+
+
+def load_driver(corpus):
+    corpus = Path(corpus)
+    if corpus.name == 'driver.py' or corpus.name == 'timestamp':
+        corpus = corpus.parent
+    sys.path.insert(0, str(corpus))
+    from driver import driver
+    sys.path.pop(0)
+    return driver
+
 
 _POOLS = {
     'thread': concurrent.futures.ThreadPoolExecutor,
@@ -26,7 +41,7 @@ def lazy_parallel(func, *iterables, **kwargs):
     worker = kwargs.get('worker', 'thread')
     max_workers = kwargs.get('max_workers')
     if max_workers is None:  # Not in back-port
-        max_workers = (multiprocessing.cpu_count() or 1)
+        max_workers = (os.cpu_count() or 1)
         if worker == 'thread':
             max_workers *= 5
     try:

tests/test_corpora.py

@@ -1,5 +1,5 @@
 import pytest
-import quantgov.corpora
+import quantgov.corpus
 import subprocess
 
 from pathlib import Path
@@ -8,15 +8,15 @@
 def build_recursive_directory_corpus(directory):
     for path, text in (('a/1.txt', u'foo'), ('b/2.txt', u'bar')):
         directory.join(path).write_text(text, encoding='utf-8', ensure=True)
-    return quantgov.corpora.RecursiveDirectoryCorpusDriver(
+    return quantgov.corpus.RecursiveDirectoryCorpusDriver(
         directory=str(directory), index_labels=('letter', 'number'))
 
 
 def build_name_pattern_corpus(directory):
     for path, text in (('a_1.txt', u'foo'), ('b_2.txt', u'bar')):
         path = directory.join(path).write_text(
             text, encoding='utf-8', ensure=True)
-    return quantgov.corpora.NamePatternCorpusDriver(
+    return quantgov.corpus.NamePatternCorpusDriver(
         pattern=r'(?P<letter>[a-z])_(?P<number>\d)',
         directory=str(directory)
     )
@@ -35,7 +35,7 @@ def build_index_corpus(directory):
     with index_path.open('w', encoding='utf-8') as outf:
         outf.write(u'letter,number,path\n')
         outf.write(u'\n'.join(','.join(row) for row in rows))
-    return quantgov.corpora.IndexDriver(str(index_path))
+    return quantgov.corpus.IndexDriver(str(index_path))
 
 
 BUILDERS = {