Init with working Router and Streamer meta-utilities.

Author: halfak

.gitignore

@@ -0,0 +1,43 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# Temporary text editor files
+*~
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+#lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Sphinx documentation
+doc/_build/
+doc/.buildfile
+*.toctree

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Aaron Halfaker
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

MANIFEST.in

@@ -0,0 +1 @@
+include LICENSE README.md requirements.txt

README.md

@@ -0,0 +1,12 @@
+# MediaWiki command-line interface
+
+A set of helper functions and classes for mediawiki-utilities command-line
+utilities. 
+
+* **Installation:** ``pip install mwcli``
+* **Documentation:** https://pythonhosted.org/mwcli
+* **Repositiory:** https://github.com/mediawiki-utilities/python-mwcli
+* **License:** MIT
+
+## Author
+* Aaron Halfaker -- https://github.com/halfak

mwcli/__init__.py

@@ -0,0 +1,4 @@
+from .streamer import Streamer, read_json
+from .router import Router
+
+__all__ = [Router, Streamer, read_json]

mwcli/errors.py

@@ -0,0 +1,5 @@
+class FileTypeError(Exception):
+    """
+    Thrown when an file is not of an expected type.
+    """
+    pass

mwcli/files/__init__.py

@@ -0,0 +1,2 @@
+from .functions import (concat, reader, writer, output_dir_path, 
+                        normalize_path, normalize_dir)

mwcli/files/functions.py

@@ -0,0 +1,175 @@
+import bz2
+import gzip
+import io
+import os
+
+from . import p7z
+from ..errors import FileTypeError
+
+FILE_READERS = {
+    'gz': lambda fn: gzip.open(fn, 'rt', encoding='utf-8', errors='replace'),
+    'bz2': lambda fn: bz2.open(fn, 'rt', encoding='utf-8', errors='replace'),
+    '7z': p7z.reader,
+    'json': lambda fn: open(fn, 'rt', encoding='utf-8', errors='replace'),
+    'xml': lambda fn: open(fn, 'rt', encoding='utf-8', errors='replace')
+}
+"""
+Maps extensions to the strategy for opening/decompressing a file
+"""
+
+FILE_WRITERS = {
+    'gz': lambda fn: gzip.open(fn, 'wt', encoding='utf-8', errors='replace'),
+    'bz2': lambda fn: bz2.open(fn, 'wt', encoding='utf-8', errors='replace'),
+    'plaintext': lambda fn: open(fn, 'wt', encoding='utf-8', errors='replace'),
+    'json': lambda fn: open(fn, 'wt', encoding='utf-8', errors='replace'),
+    'xml': lambda fn: open(fn, 'wt', encoding='utf-8', errors='replace')
+}
+"""
+Maps compression types to the strategy for opening/compressing a file
+"""
+
+
+def extract_extension(path):
+    """
+    Reads a file path and returns the extension or None if the path
+    contains no extension.
+
+    :Parameters:
+        path : str
+            A filesystem path
+    """
+    filename = os.path.basename(path)
+    parts = filename.split(".")
+    if len(parts) == 1:
+        return filename, None
+    else:
+        return ".".join(parts[:-1]), parts[-1]
+
+
+def normalize_path(path_or_f):
+    """
+    Verifies that a file exists at a given path and that the file has a
+    known extension type.
+
+    :Parameters:
+        path_or_f : `str` | `file`
+            the path to a dump file or a file handle
+
+    """
+    if hasattr(path_or_f, "read"):
+        return path_or_f
+    else:
+        path = path_or_f
+
+    path = os.path.expanduser(path)
+
+    # Check if exists and is a file
+    if os.path.isdir(path):
+        raise IsADirectoryError("Is a directory: {0}".format(path))
+    elif not os.path.isfile(path):
+        raise FileNotFoundError("No such file: {0}".format(path))
+
+    _, extension = extract_extension(path)
+
+    if extension not in FILE_READERS:
+        raise FileTypeError("Extension {0} is not supported."
+                            .format(repr(extension)))
+
+    return path
+
+
+def normalize_dir(path):
+    if os.path.exists(path) and not os.path.isdir(path):
+        raise NotADirectoryError("Not a directory: {0}".format(path))
+    else:
+        os.makedirs(path, exist_ok=True)
+
+    return path
+
+
+def reader(path_or_f):
+    """
+    Turns a path to a compressed file into a file-like object of (decompressed)
+    data.
+
+    :Parameters:
+        path : `str`
+            the path to the dump file to read
+    """
+    if hasattr(path_or_f, "read"):
+        return path_or_f
+    else:
+        path = path_or_f
+
+    path = normalize_path(path)
+    _, extension = extract_extension(path)
+
+    reader_func = FILE_READERS[extension]
+
+    return reader_func(path)
+
+
+def output_dir_path(old_path, output_dir, compression):
+    filename, extension = extract_extension(old_path)
+    new_filename = filename + "." + compression
+    return os.path.join(output_dir, new_filename)
+
+
+def writer(path):
+    """
+    Creates a compressed file writer from for a path with a specified
+    compression type.
+    """
+    filename, extension = extract_extension(path)
+    if extension in FILE_WRITERS:
+        writer_func = FILE_WRITERS[extension]
+        return writer_func(path)
+    else:
+        raise RuntimeError("Output compression {0} not supported.  Type {1}"
+                           .format(extension, tuple(FILE_WRITERS.keys())))
+
+
+class ConcatinatingTextReader(io.TextIOBase):
+
+    def __init__(self, *items):
+        self.items = [io.StringIO(i) if isinstance(i, str) else i
+                      for i in items]
+
+    def read(self, size=-1):
+        return "".join(self._read(size))
+
+    def readline(self):
+
+        if len(self.items) > 0:
+            line = self.items[0].readline()
+            if line == "":
+                self.items.pop(0)
+        else:
+            line = ""
+
+        return line
+
+    def _read(self, size):
+        if size > 0:
+            while len(self.items) > 0:
+                byte_vals = self.items[0].read(size)
+                yield byte_vals
+                if len(byte_vals) < size:
+                    size = size - len(byte_vals)  # Decrement bytes
+                    self.items.pop(0)
+                else:
+                    break
+
+        else:
+            for item in self.items:
+                yield item.read()
+
+
+def concat(*stream_items):
+    """
+    Performs a streaming concatenation of `str` or `file`.
+    :Parameters:
+        \*stream_items : `str` | `file`
+            A list of items to concatenate together
+    """
+    return ConcatinatingTextReader(*stream_items)

mwcli/files/p7z.py

@@ -0,0 +1,23 @@
+import io
+import os
+import subprocess
+
+file_open = open
+
+
+def reader(path):
+    """
+    Turns a path to a dump file into a file-like object of (decompressed)
+    XML data assuming that '7z' is installed and will know what to do.
+
+    :Parameters:
+        path : `str`
+            the path to the dump file to read
+    """
+    p = subprocess.Popen(
+        ['7z', 'e', '-so', path],
+        stdout=subprocess.PIPE,
+        stderr=file_open(os.devnull, "w")
+    )
+    return io.TextIOWrapper(p.stdout, encoding='utf-8',
+                            errors='replace')

mwcli/files/tests/__init__.py

@@ -0,0 +1,43 @@
+import io
+import os
+
+from nose.tools import eq_, raises
+
+from ...errors import FileTypeError
+from ..functions import concat, extract_extension, normalize_path, reader
+
+
+def test_concat():
+    start = "Foobar1"
+    f = io.StringIO("Foobar2")
+    end = "Foobar3"
+
+    eq_(concat(start, f, end).read(), "Foobar1Foobar2Foobar3")
+
+
+def test_extract_extension():
+    eq_(extract_extension("foo")[1], None)
+    eq_(extract_extension("foo.xml")[1], "xml")
+    eq_(extract_extension("foo.xml.gz")[1], "gz")
+    eq_(extract_extension("foo.xml.bz2")[1], "bz2")
+    eq_(extract_extension("foo.xml-p10001p10200.7z")[1], "7z")
+
+
+@raises(FileNotFoundError)
+def test_normalize_path_noexist():
+    normalize_path("IDONTEXIST!!!")
+
+
+@raises(IsADirectoryError)
+def test_normalize_path_directory():
+    normalize_path(os.path.dirname(__file__))
+
+
+@raises(FileTypeError)
+def test_normalize_path_bad_extension():
+    normalize_path(__file__)
+
+
+def test_open():
+    f = io.StringIO()
+    eq_(f, reader(f))

mwcli/files/tests/foo.7z

@@ -0,0 +1,10 @@
+import os.path
+
+from nose.tools import eq_
+
+from ..p7z import reader
+
+
+def test_open_file():
+    f = reader(os.path.join(os.path.dirname(__file__), "foo.7z"))
+    eq_(f.read(), "foo\n")