diff --git a/.gitignore b/.gitignore index f2cf27245..6de62bd55 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ docs/_build *.orig .coverage +example.db +*.cfg diff --git a/default-sample.cfg b/default-sample.cfg index 4d96cc327..db45784a9 100644 --- a/default-sample.cfg +++ b/default-sample.cfg @@ -28,9 +28,10 @@ maxrequestsize=3mb url=http://localhost:5000/wps outputurl=http://localhost:5000/outputs/ outputpath=outputs -workdir=workdir +workdir=/tmp maxprocesses=10 parallelprocesses=2 +store_type=db [processing] mode=default @@ -44,3 +45,15 @@ format=%(asctime)s] [%(levelname)s] file=%(pathname)s line=%(lineno)s module=%(m [grass] gisbase=/usr/local/grass-7.3.svn/ + + +[db] +db_type=pg +dbname=dbname +user=username +password=password +host=localhost +port=5432 +dblocation=/tmp/db.sqlite +schema_name=test_schema + diff --git a/pywps/configuration.py b/pywps/configuration.py index 6ac8351b0..bf8fff395 100755 --- a/pywps/configuration.py +++ b/pywps/configuration.py @@ -131,6 +131,8 @@ def load_configuration(cfgfiles=None): CONFIG.add_section('grass') CONFIG.set('grass', 'gisbase', '') + CONFIG.add_section('db') + if not cfgfiles: cfgfiles = _get_default_config_files_location() diff --git a/pywps/inout/formats/__init__.py b/pywps/inout/formats/__init__.py index 9fbf51f56..3acffa4c0 100644 --- a/pywps/inout/formats/__init__.py +++ b/pywps/inout/formats/__init__.py @@ -11,16 +11,29 @@ # based on Web Processing Service Best Practices Discussion Paper, OGC 12-029 # http://opengeospatial.org/standards/wps +from enum import Enum from collections import namedtuple import mimetypes from pywps.validator.mode import MODE from pywps.validator.base import emptyvalidator - _FORMATS = namedtuple('FORMATS', 'GEOJSON, JSON, SHP, GML, GEOTIFF, WCS,' 'WCS100, WCS110, WCS20, WFS, WFS100,' 'WFS110, WFS20, WMS, WMS130, WMS110,' - 'WMS100, TEXT, NETCDF, LAZ, LAS') + 'WMS100, TEXT, CSV, NETCDF, LAZ, LAS') + + +# this should be Enum type (only compatible with Python 3) +class DATA_TYPE(Enum): + VECTOR = 0 + RASTER = 1 + OTHER = 2 + + def is_valid_datatype(data_type): + + known_values = [datatype for datatype in DATA_TYPE] + if data_type not in known_values: + raise Exception("Unknown data type") class Format(object): @@ -39,7 +52,7 @@ class Format(object): def __init__(self, mime_type, schema=None, encoding=None, validate=emptyvalidator, mode=MODE.SIMPLE, - extension=None): + extension=None, data_type=None): """Constructor """ @@ -47,12 +60,14 @@ def __init__(self, mime_type, self._encoding = None self._schema = None self._extension = None + self._data_type = None self.mime_type = mime_type self.encoding = encoding self.schema = schema self.validate = validate self.extension = extension + self.data_type = data_type @property def mime_type(self): @@ -62,6 +77,20 @@ def mime_type(self): return self._mime_type + @property + def data_type(self): + """Get format data type + """ + + return self._data_type + + @data_type.setter + def data_type(self, data_type): + """Set format encoding + """ + + self._data_type = data_type + @mime_type.setter def mime_type(self, mime_type): """Set format mime type @@ -143,7 +172,8 @@ def json(self): 'mime_type': self.mime_type, 'encoding': self.encoding, 'schema': self.schema, - 'extension': self.extension + 'extension': self.extension, + 'data_type': self.data_type } @json.setter @@ -156,30 +186,32 @@ def json(self, jsonin): self.encoding = jsonin['encoding'] self.schema = jsonin['schema'] self.extension = jsonin['extension'] + self.data_type = jsonin['data_type'] FORMATS = _FORMATS( - Format('application/vnd.geo+json', extension='.geojson'), - Format('application/json', extension='.json'), - Format('application/x-zipped-shp', extension='.zip'), - Format('application/gml+xml', extension='.gml'), - Format('image/tiff; subtype=geotiff', extension='.tiff'), - Format('application/xogc-wcs', extension='.xml'), - Format('application/x-ogc-wcs; version=1.0.0', extension='.xml'), - Format('application/x-ogc-wcs; version=1.1.0', extension='.xml'), - Format('application/x-ogc-wcs; version=2.0', extension='.xml'), - Format('application/x-ogc-wfs', extension='.xml'), - Format('application/x-ogc-wfs; version=1.0.0', extension='.xml'), - Format('application/x-ogc-wfs; version=1.1.0', extension='.xml'), - Format('application/x-ogc-wfs; version=2.0', extension='.xml'), - Format('application/x-ogc-wms', extension='.xml'), - Format('application/x-ogc-wms; version=1.3.0', extension='.xml'), - Format('application/x-ogc-wms; version=1.1.0', extension='.xml'), - Format('application/x-ogc-wms; version=1.0.0', extension='.xml'), - Format('text/plain', extension='.txt'), - Format('application/x-netcdf', extension='.nc'), - Format('application/octet-stream', extension='.laz'), - Format('application/octet-stream', extension='.las'), + Format('application/vnd.geo+json', extension='.geojson', data_type=DATA_TYPE.VECTOR), + Format('application/json', extension='.json', data_type=DATA_TYPE.VECTOR), + Format('application/x-zipped-shp', extension='.zip', data_type=DATA_TYPE.VECTOR), + Format('application/gml+xml', extension='.gml', data_type=DATA_TYPE.VECTOR), + Format('image/tiff; subtype=geotiff', extension='.tiff', data_type=DATA_TYPE.RASTER), + Format('application/xogc-wcs', extension='.xml', data_type=DATA_TYPE.VECTOR), + Format('application/x-ogc-wcs; version=1.0.0', extension='.xml', data_type=DATA_TYPE.VECTOR), + Format('application/x-ogc-wcs; version=1.1.0', extension='.xml', data_type=DATA_TYPE.VECTOR), + Format('application/x-ogc-wcs; version=2.0', extension='.xml', data_type=DATA_TYPE.VECTOR), + Format('application/x-ogc-wfs', extension='.xml', data_type=DATA_TYPE.VECTOR), + Format('application/x-ogc-wfs; version=1.0.0', extension='.xml', data_type=DATA_TYPE.VECTOR), + Format('application/x-ogc-wfs; version=1.1.0', extension='.xml', data_type=DATA_TYPE.VECTOR), + Format('application/x-ogc-wfs; version=2.0', extension='.xml', data_type=DATA_TYPE.VECTOR), + Format('application/x-ogc-wms', extension='.xml', data_type=DATA_TYPE.VECTOR), + Format('application/x-ogc-wms; version=1.3.0', extension='.xml', data_type=DATA_TYPE.VECTOR), + Format('application/x-ogc-wms; version=1.1.0', extension='.xml', data_type=DATA_TYPE.VECTOR), + Format('application/x-ogc-wms; version=1.0.0', extension='.xml', data_type=DATA_TYPE.VECTOR), + Format('text/plain', extension='.txt', data_type=DATA_TYPE.OTHER), + Format('text/csv', extension='.csv', data_type=DATA_TYPE.OTHER), + Format('application/x-netcdf', extension='.nc', data_type=DATA_TYPE.VECTOR), + Format('application/octet-stream', extension='.laz', data_type=DATA_TYPE.VECTOR), + Format('application/octet-stream', extension='.las', data_type=DATA_TYPE.VECTOR), ) diff --git a/pywps/inout/outputs.py b/pywps/inout/outputs.py index 092a96633..f30b58572 100644 --- a/pywps/inout/outputs.py +++ b/pywps/inout/outputs.py @@ -9,7 +9,9 @@ import lxml.etree as etree import six from pywps.inout import basic -from pywps.inout.storage import FileStorage +from pywps.inout.storage.file import FileStorage +from pywps.inout.storage.db import DbStorage +from pywps import configuration as config from pywps.validator.mode import MODE @@ -99,9 +101,21 @@ def _json_reference(self, data): data["type"] = "reference" # get_url will create the file and return the url for it - self.storage = FileStorage() data["href"] = self.get_url() + store_type = config.get_config_value('server', 'store_type') + self.storage = None + if store_type == 'db': + db_storage_instance = DbStorage() + self.storage = db_storage_instance.get_db_type() + else: + self.storage = FileStorage() + + #to be implemented: + #elif store_type == 's3' and \ + # config.get_config_value('s3', 'bucket_name'): + # self.storage = S3Storage() + if self.data_format: if self.data_format.mime_type: data['mimetype'] = self.data_format.mime_type diff --git a/pywps/inout/storage/__init__.py b/pywps/inout/storage/__init__.py new file mode 100644 index 000000000..3110d224c --- /dev/null +++ b/pywps/inout/storage/__init__.py @@ -0,0 +1,46 @@ +################################################################## +# Copyright 2018 Open Source Geospatial Foundation and others # +# licensed under MIT, Please consult LICENSE.txt for details # +################################################################## + +from abc import ABCMeta, abstractmethod + + +class STORE_TYPE: + PATH = 0 + DB = 1 + + +class StorageAbstract(object): + """Data storage abstract class + """ + + __metaclass__ = ABCMeta + + @abstractmethod + def store(self, output): + """ + :param output: of type IOHandler + :returns: (type, store, url) where + type - is type of STORE_TYPE - number + store - string describing storage - file name, database connection + url - url, where the data can be downloaded + """ + pass + + +class DummyStorage(StorageAbstract): + """Dummy empty storage implementation, does nothing + + Default instance, for non-reference output request + + >>> store = DummyStorage() + >>> assert store.store + """ + + def __init__(self): + """ + """ + + def store(self, output): + pass diff --git a/pywps/inout/storage/db/__init__.py b/pywps/inout/storage/db/__init__.py new file mode 100644 index 000000000..b7bac9c52 --- /dev/null +++ b/pywps/inout/storage/db/__init__.py @@ -0,0 +1,58 @@ +################################################################## +# Copyright 2018 Open Source Geospatial Foundation and others # +# licensed under MIT, Please consult LICENSE.txt for details # +################################################################## + +import logging +from abc import ABCMeta, abstractmethod +from pywps import configuration as config +from pywps.inout.formats import DATA_TYPE +from pywps.exceptions import NoApplicableCode +from .. import STORE_TYPE +from .. import StorageAbstract +import sqlalchemy + + +LOGGER = logging.getLogger('PYWPS') + + +class DbStorage(StorageAbstract): + + def __init__(self): + # get db_type from configuration + try: + self.db_type = config.get_config_value('db', 'db_type').lower() + except KeyError: + raise Exception("Database type has not been specified") + + @staticmethod + def get_db_type(): + from . import sqlite + from . import pg + # create an instance of the appropriate class + db_type = config.get_config_value('db', 'db_type').lower() + if db_type == "pg": + storage = pg.PgStorage() + elif db_type == "sqlite": + storage = sqlite.SQLiteStorage() + else: + raise Exception("Unknown database type: '{}'".format(config.get_config_value('db', 'db_type').lower())) + + return storage + + def initdb(self): + pass + + def store(self, output): + """ Creates reference that is returned to the client + """ + pass + + def store_vector_output(self, file_name, identifier): + pass + + def store_raster_output(self, file_name, identifier): + pass + + def store_other_output(self, file_name, identifier, uuid): + pass diff --git a/pywps/inout/storage/db/pg.py b/pywps/inout/storage/db/pg.py new file mode 100644 index 000000000..e6413b6f3 --- /dev/null +++ b/pywps/inout/storage/db/pg.py @@ -0,0 +1,164 @@ +################################################################## +# Copyright 2018 Open Source Geospatial Foundation and others # +# licensed under MIT, Please consult LICENSE.txt for details # +################################################################## + +import logging +from pywps import configuration as config +from pywps.exceptions import NoApplicableCode +from .. import STORE_TYPE +from pywps.inout.formats import DATA_TYPE +from . import DbStorage +import sqlalchemy + +LOGGER = logging.getLogger('PYWPS') + + +class PgStorage(DbStorage): + + def __init__(self): + # TODO: more databases in config file + # create connection string + dbsettings = "db" + self.dbname = config.get_config_value(dbsettings, "dbname") + self.user = config.get_config_value(dbsettings, "user") + self.password = config.get_config_value(dbsettings, "password") + self.host = config.get_config_value(dbsettings, "host") + self.port = config.get_config_value(dbsettings, "port") + + self.target = "dbname={} user={} password={} host={} port={}".format( + self.dbname, self.user, self.password, self.host, self.port + ) + + self.schema_name = config.get_config_value(dbsettings, "schema_name") + + self.initdb() + + def initdb(self): + + from sqlalchemy.schema import CreateSchema + + dbsettings = "db" + connstr = 'postgresql://{}:{}@{}:{}/{}'.format( + config.get_config_value(dbsettings, "user"), + config.get_config_value(dbsettings, "password"), + config.get_config_value(dbsettings, "host"), + config.get_config_value(dbsettings, "port"), + config.get_config_value(dbsettings, "dbname") + ) + + engine = sqlalchemy.create_engine(connstr) + schema_name = config.get_config_value('db', 'schema_name') + + # Create schema; if it already exists, skip this + try: + engine.execute(CreateSchema(schema_name)) + # programming error - schema already exists) + except sqlalchemy.exc.ProgrammingError: + pass + + def store_vector_output(self, file_name, identifier): + """ Open output file, connect to PG database and copy data there + """ + from osgeo import ogr + + db_location = self.schema_name + "." + identifier + dsc_out = ogr.Open("PG:" + self.target) + + # connect to a database and copy output there + LOGGER.debug("Database: {}".format(self.target)) + dsc_in = ogr.Open(file_name) + if dsc_in is None: + raise Exception("Reading data failed.") + if dsc_out is None: + raise NoApplicableCode("Could not connect to the database.") + layer = dsc_out.CopyLayer(dsc_in.GetLayer(), db_location, + ['OVERWRITE=YES']) + + if layer is None: + raise Exception("Writing output data to the database failed.") + + dsc_out.Destroy() + dsc_in.Destroy() + + # returns process identifier (defined within the process) + return identifier + + def store_raster_output(self, file_name, identifier): + + from subprocess import run, Popen, PIPE + + # Convert raster to an SQL query + command1 = ["raster2pgsql", "-a", file_name, self.schema_name + "." + identifier] + p = Popen(command1, stdout=PIPE) + # Apply the SQL query + command2 = ["psql", "-h", "localhost", "-p", "5432", "-d", self.dbname] + run(command2, stdin=p.stdout) + + return identifier + + def store_other_output(self, file_name, identifier, uuid): + + from sqlalchemy import Column, Integer, String, LargeBinary, DateTime, func, create_engine + from sqlalchemy.ext.declarative import declarative_base + from sqlalchemy.orm import sessionmaker + + base = declarative_base() + + engine = create_engine( + 'postgresql://{}:{}@{}:{}/{}'.format( + self.dbname, + self.password, + self.host, + self.port, + self.user + ) + ) + + # Create table + class Other_output(base): + __tablename__ = identifier + __table_args__ = {'schema': self.schema_name} + + primary_key = Column(Integer, primary_key=True) + uuid = Column(String(64)) + data = Column(LargeBinary) + timestamp = Column(DateTime(timezone=True), server_default=func.now()) + + Session = sessionmaker(engine) + session = Session() + + base.metadata.create_all(engine) + + # Open file as binary + with open(file_name, "rb") as data: + out = data.read() + + # Add data to table + output = Other_output(uuid=uuid, data=out) + session.add(output) + session.commit() + + return identifier + + def store(self, output): + """ Creates reference that is returned to the client + """ + + DATA_TYPE.is_valid_datatype(output.output_format.data_type) + + if output.output_format.data_type is DATA_TYPE.VECTOR: + self.store_vector_output(output.file, output.identifier) + elif output.output_format.data_type is DATA_TYPE.RASTER: + self.store_raster_output(output.file, output.identifier) + elif output.output_format.data_type is DATA_TYPE.OTHER: + self.store_other_output(output.file, output.identifier, output.uuid) + else: + # This should never happen + raise Exception("Unknown data type") + + url = '{}.{}.{}'.format(self.dbname, self.schema_name, output.identifier) + + # returns value for database storage defined in the STORE_TYPE class, + # name of the output file and a reference + return (STORE_TYPE.DB, output.file, url) diff --git a/pywps/inout/storage/db/sqlite.py b/pywps/inout/storage/db/sqlite.py new file mode 100644 index 000000000..9499e4825 --- /dev/null +++ b/pywps/inout/storage/db/sqlite.py @@ -0,0 +1,113 @@ +################################################################## +# Copyright 2018 Open Source Geospatial Foundation and others # +# licensed under MIT, Please consult LICENSE.txt for details # +################################################################## + +import logging +from pywps import configuration as config +from .. import STORE_TYPE +from pywps.inout.formats import DATA_TYPE +from pywps.exceptions import NoApplicableCode +from . import DbStorage + +LOGGER = logging.getLogger('PYWPS') + + +class SQLiteStorage(DbStorage): + + def __init__(self): + + self.target = config.get_config_value("db", "dblocation") + + def store_vector_output(self, file_name, identifier): + """ Open output file, connect to SQLite database and copiy data there + """ + from osgeo import ogr + + drv = ogr.GetDriverByName("SQLite") + dsc_out = drv.CreateDataSource(self.target) + + # connect to a database and copy output there + LOGGER.debug("Database: {}".format(self.target)) + dsc_in = ogr.Open(file_name) + if dsc_in is None: + raise Exception("Reading data failed.") + if dsc_out is None: + raise NoApplicableCode("Could not connect to the database.") + layer = dsc_out.CopyLayer(dsc_in.GetLayer(), identifier, + ['OVERWRITE=YES']) + + if layer is None: + raise Exception("Writing output data to the database failed.") + + dsc_out.Destroy() + dsc_in.Destroy() + + # returns process identifier (defined within the process) + return identifier + + def store_raster_output(self, file_name, identifier): + + from subprocess import call + + call(["gdal_translate", "-of", "Rasterlite", file_name, "RASTERLITE:" + self.target + ",table=" + identifier]) + + # returns process identifier (defined within the process) + return identifier + + def store_other_output(self, file_name, identifier, uuid): + + from sqlalchemy import Column, Integer, String, LargeBinary, DateTime, func, create_engine + from sqlalchemy.ext.declarative import declarative_base + from sqlalchemy.orm import sessionmaker + + base = declarative_base() + + engine = create_engine("sqlite:///{}".format(self.target)) + + # Create table + class Other_output(base): + __tablename__ = identifier + + primary_key = Column(Integer, primary_key=True) + uuid = Column(String(64)) + data = Column(LargeBinary) + timestamp = Column(DateTime(timezone=True), server_default=func.now()) + + Session = sessionmaker(engine) + session = Session() + + base.metadata.create_all(engine) + + # Open file as binary + with open(file_name, "rb") as data: + out = data.read() + + # Add data to table + output = Other_output(uuid=uuid, data=out) + session.add(output) + session.commit() + + return identifier + + def store(self, output): + """ Creates reference that is returned to the client + """ + + DATA_TYPE.is_valid_datatype(output.output_format.data_type) + + if output.output_format.data_type is DATA_TYPE.VECTOR: + self.store_vector_output(output.file, output.identifier) + elif output.output_format.data_type is DATA_TYPE.RASTER: + self.store_raster_output(output.file, output.identifier) + elif output.output_format.data_type is DATA_TYPE.OTHER: + self.store_other_output(output.file, output.identifier, output.uuid) + else: + # This should never happen + raise Exception("Unknown data type") + + url = '{}.{}'.format(self.target, output.identifier) + + # returns value for database storage defined in the STORE_TYPE class, + # name of the output file and a reference + return (STORE_TYPE.DB, output.file, url) diff --git a/pywps/inout/storage.py b/pywps/inout/storage/file.py similarity index 81% rename from pywps/inout/storage.py rename to pywps/inout/storage/file.py index 53e37c6de..9d5e1102b 100644 --- a/pywps/inout/storage.py +++ b/pywps/inout/storage/file.py @@ -5,54 +5,14 @@ import logging import os -from abc import ABCMeta, abstractmethod from pywps._compat import urljoin from pywps.exceptions import NotEnoughStorage from pywps import configuration as config +from . import StorageAbstract, STORE_TYPE LOGGER = logging.getLogger('PYWPS') -class STORE_TYPE: - PATH = 0 -# TODO: cover with tests - - -class StorageAbstract(object): - """Data storage abstract class - """ - - __metaclass__ = ABCMeta - - @abstractmethod - def store(self, output): - """ - :param output: of type IOHandler - :returns: (type, store, url) where - type - is type of STORE_TYPE - number - store - string describing storage - file name, database connection - url - url, where the data can be downloaded - """ - pass - - -class DummyStorage(StorageAbstract): - """Dummy empty storage implementation, does nothing - - Default instance, for non-reference output request - - >>> store = DummyStorage() - >>> assert store.store - """ - - def __init__(self): - """ - """ - - def store(self, ouput): - pass - - class FileStorage(StorageAbstract): """File storage implementation, stores data to file system diff --git a/tests/data/other/corn.csv b/tests/data/other/corn.csv new file mode 100644 index 000000000..58245d350 --- /dev/null +++ b/tests/data/other/corn.csv @@ -0,0 +1,106 @@ +"Program","Year","Period","Week Ending","Geo Level","State","State ANSI","Ag District","Ag District Code","County","County ANSI","Zip Code","Region","watershed_code","Watershed","Commodity","Data Item","Domain","Domain Category","Value","CV (%)" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","CENTRAL","50","BARTON","009","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","23,286","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","CENTRAL","50","DICKINSON","041","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","13,452","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","CENTRAL","50","ELLIS","051","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","1,406","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","CENTRAL","50","ELLSWORTH","053","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","2,284","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","CENTRAL","50","LINCOLN","105","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","2,094","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","CENTRAL","50","MARION","115","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","35,242","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","CENTRAL","50","MCPHERSON","113","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","39,661","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","CENTRAL","50","RICE","159","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","30,347","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","CENTRAL","50","RUSH","165","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","7,735","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","CENTRAL","50","RUSSELL","167","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","3,801","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","CENTRAL","50","SALINE","169","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","4,776","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","EAST CENTRAL","80","ANDERSON","003","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","56,288","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","EAST CENTRAL","80","CHASE","017","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","4,661","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","EAST CENTRAL","80","COFFEY","031","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","27,447","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","EAST CENTRAL","80","DOUGLAS","045","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","31,483","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","EAST CENTRAL","80","FRANKLIN","059","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","32,489","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","EAST CENTRAL","80","GEARY","061","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","9,634","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","EAST CENTRAL","80","JOHNSON","091","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","10,818","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","EAST CENTRAL","80","LINN","107","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","21,805","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","EAST CENTRAL","80","LYON","111","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","25,472","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","EAST CENTRAL","80","MIAMI","121","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","30,557","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","EAST CENTRAL","80","MORRIS","127","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","13,255","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","EAST CENTRAL","80","OSAGE","139","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","51,759","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","EAST CENTRAL","80","SHAWNEE","177","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","37,779","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","EAST CENTRAL","80","WABAUNSEE","197","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","14,776","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTH CENTRAL","40","CLAY","027","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","32,427","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTH CENTRAL","40","CLOUD","029","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","22,658","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTH CENTRAL","40","JEWELL","089","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","31,590","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTH CENTRAL","40","MITCHELL","123","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","15,838","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTH CENTRAL","40","OSBORNE","141","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","7,811","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTH CENTRAL","40","OTTAWA","143","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","10,603","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTH CENTRAL","40","PHILLIPS","147","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","27,554","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTH CENTRAL","40","REPUBLIC","157","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","64,432","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTH CENTRAL","40","ROOKS","163","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","7,146","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTH CENTRAL","40","SMITH","183","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","33,082","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTH CENTRAL","40","WASHINGTON","201","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","56,372","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTHEAST","70","ATCHISON","005","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","57,143","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTHEAST","70","BROWN","013","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","102,394","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTHEAST","70","DONIPHAN","043","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","67,945","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTHEAST","70","JACKSON","085","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","23,097","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTHEAST","70","JEFFERSON","087","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","33,661","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTHEAST","70","LEAVENWORTH","103","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","15,751","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTHEAST","70","MARSHALL","117","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","83,928","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTHEAST","70","NEMAHA","131","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","89,818","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTHEAST","70","POTTAWATOMIE","149","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","35,558","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTHEAST","70","RILEY","161","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","11,438","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTHEAST","70","WYANDOTTE","209","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","1,741","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTHWEST","10","CHEYENNE","023","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","53,256","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTHWEST","10","DECATUR","039","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","79,490","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTHWEST","10","GRAHAM","065","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","28,495","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTHWEST","10","NORTON","137","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","67,620","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTHWEST","10","RAWLINS","153","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","66,074","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTHWEST","10","SHERIDAN","179","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","123,299","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTHWEST","10","SHERMAN","181","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","108,802","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","NORTHWEST","10","THOMAS","193","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","171,616","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTH CENTRAL","60","BARBER","007","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","6,736","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTH CENTRAL","60","COMANCHE","033","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","1,921","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTH CENTRAL","60","EDWARDS","047","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","74,394","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTH CENTRAL","60","HARPER","077","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","4,384","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTH CENTRAL","60","HARVEY","079","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","57,651","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTH CENTRAL","60","KINGMAN","095","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","12,877","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTH CENTRAL","60","KIOWA","097","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","23,458","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTH CENTRAL","60","PAWNEE","145","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","38,920","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTH CENTRAL","60","PRATT","151","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","56,145","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTH CENTRAL","60","RENO","155","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","30,106","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTH CENTRAL","60","SEDGWICK","173","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","37,730","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTH CENTRAL","60","STAFFORD","185","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","56,586","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTH CENTRAL","60","SUMNER","191","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","35,362","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHEAST","90","ALLEN","001","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","23,937","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHEAST","90","BOURBON","011","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","13,622","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHEAST","90","BUTLER","015","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","69,751","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHEAST","90","CHAUTAUQUA","019","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","3,100","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHEAST","90","CHEROKEE","021","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","54,289","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHEAST","90","COWLEY","035","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","17,558","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHEAST","90","CRAWFORD","037","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","51,857","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHEAST","90","ELK","049","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","4,108","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHEAST","90","GREENWOOD","073","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","10,716","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHEAST","90","LABETTE","099","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","42,635","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHEAST","90","MONTGOMERY","125","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","34,990","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHEAST","90","NEOSHO","133","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","36,161","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHEAST","90","WILSON","205","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","36,376","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHEAST","90","WOODSON","207","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","30,280","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHWEST","30","CLARK","025","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","1,285","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHWEST","30","FINNEY","055","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","92,465","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHWEST","30","FORD","057","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","41,013","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHWEST","30","GRANT","067","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","47,834","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHWEST","30","GRAY","069","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","58,589","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHWEST","30","HAMILTON","075","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","10,780","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHWEST","30","HASKELL","081","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","60,344","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHWEST","30","HODGEMAN","083","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","11,093","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHWEST","30","KEARNY","093","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","22,959","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHWEST","30","MEADE","119","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","94,825","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHWEST","30","MORTON","129","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","28,394","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHWEST","30","SEWARD","175","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","50,062","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHWEST","30","STANTON","187","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","57,241","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","SOUTHWEST","30","STEVENS","189","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","115,242","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","WEST CENTRAL","20","GOVE","063","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","76,031","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","WEST CENTRAL","20","GREELEY","071","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","31,939","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","WEST CENTRAL","20","LANE","101","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","11,024","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","WEST CENTRAL","20","LOGAN","109","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","58,078","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","WEST CENTRAL","20","NESS","135","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","3,679","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","WEST CENTRAL","20","SCOTT","171","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","34,315","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","WEST CENTRAL","20","TREGO","195","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","7,589","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","WEST CENTRAL","20","WALLACE","199","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","64,455","15.4" +"CENSUS","2012","YEAR","","COUNTY","KANSAS","20","WEST CENTRAL","20","WICHITA","203","","","00000000","","CORN","CORN, GRAIN - ACRES HARVESTED","TOTAL","NOT SPECIFIED","40,630","15.4" diff --git a/tests/data/other/test.txt b/tests/data/other/test.txt new file mode 100644 index 000000000..be1c84eaa --- /dev/null +++ b/tests/data/other/test.txt @@ -0,0 +1,2 @@ +"Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum." + diff --git a/tests/test_formats.py b/tests/test_formats.py index 1140e9d88..9541568da 100644 --- a/tests/test_formats.py +++ b/tests/test_formats.py @@ -83,6 +83,7 @@ def test_json_in(self): injson['extension'] = '.gml' injson['mime_type'] = 'application/gml+xml' injson['encoding'] = 'utf-8' + injson['data_type'] = 'vector' frmt = Format(injson['mime_type']) frmt.json = injson diff --git a/tests/test_storage.py b/tests/test_storage.py new file mode 100644 index 000000000..224431d4b --- /dev/null +++ b/tests/test_storage.py @@ -0,0 +1,362 @@ +import unittest +import atexit +import shutil +import tempfile +from sqlalchemy import create_engine, inspect +from pywps import FORMATS +from pywps.inout.storage import DummyStorage, STORE_TYPE +from pywps.inout.storage.file import FileStorage +from pywps.inout.storage.db.pg import PgStorage +from pywps.inout.storage.db.sqlite import SQLiteStorage +from pywps import ComplexOutput +import os +from pywps import configuration + + + +TEMP_DIRS=[] + +def clear(): + """Delete temporary files + """ + for d in TEMP_DIRS: + shutil.rmtree(d) + +atexit.register(clear) + +def get_vector_file(): + + return os.path.join(os.path.dirname(__file__), "data", "gml", "point.gml") + +def get_raster_file(): + + return os.path.join(os.path.dirname(__file__), "data", "geotiff", "dem.tiff") + + +def get_text_file(): + + return os.path.join(os.path.dirname(__file__), "data", "other", "test.txt") + +def get_csv_file(): + + return os.path.join(os.path.dirname(__file__), "data", "other", "corn.csv") + + + + +def set_test_configuration(): + configuration.CONFIG.set("server", "store_type", "db") + #when add_section('db') -> duplicate error, section db exists ; if not -> no section db ; section created in configuration.py + #configuration.CONFIG.add_section('db') + configuration.CONFIG.set("db", "db_type", "pg") + configuration.CONFIG.set("db", "dbname", "pisl") + configuration.CONFIG.set("db", "user", "pisl") + configuration.CONFIG.set("db", "password", "password") + configuration.CONFIG.set("db", "host", "localhost") + configuration.CONFIG.set("db", "port", "5432") + configuration.CONFIG.set("db", "schema_name", "test_schema") + + +class DummyStorageTestCase(unittest.TestCase): + """Storage test case + """ + + def setUp(self): + global TEMP_DIRS + tmp_dir = tempfile.mkdtemp() + TEMP_DIRS.append(tmp_dir) + + self.storage = DummyStorage() + + def tearDown(self): + pass + + def test_dummy_storage(self): + assert isinstance(self.storage, DummyStorage) + + + def test_store(self): + vector_output = ComplexOutput('vector', 'Vector output', + supported_formats=[FORMATS.GML]) + vector_output.file = get_vector_file() + assert not self.storage.store("some data") + + +class FileStorageTestCase(unittest.TestCase): + """FileStorage tests + """ + + def setUp(self): + global TEMP_DIRS + tmp_dir = tempfile.mkdtemp() + TEMP_DIRS.append(tmp_dir) + + self.storage = FileStorage() + + def tearDown(self): + pass + + def test_file_storage(self): + assert isinstance(self.storage, FileStorage) + + + def test_store(self): + vector_output = ComplexOutput('vector', 'Vector output', + supported_formats=[FORMATS.GML]) + vector_output.file = get_vector_file() + + store_file = self.storage.store(vector_output) + assert len(store_file) == 3 + assert store_file[0] == STORE_TYPE.PATH + assert isinstance(store_file[1], str) + assert isinstance(store_file[2], str) + + +class PgStorageTestCase(unittest.TestCase): + """PgStorage test + """ + + def setUp(self): + global TEMP_DIRS + tmp_dir = tempfile.mkdtemp() + TEMP_DIRS.append(tmp_dir) + set_test_configuration() + self.storage = PgStorage() + + dbsettings = "db" + self.dbname = configuration.get_config_value(dbsettings, "dbname") + self.user = configuration.get_config_value(dbsettings, "user") + self.password = configuration.get_config_value(dbsettings, "password") + self.host = configuration.get_config_value(dbsettings, "host") + self.port = configuration.get_config_value(dbsettings, "port") + + self.storage.target = "dbname={} user={} password={} host={} port={}".format( + self.dbname, self.user, self.password, self.host, self.port + ) + + self.storage.schema_name = configuration.get_config_value("db", "schema_name") + self.storage.dbname = configuration.get_config_value("db", "dbname") + + def tearDown(self): + pass + + def test_pg_storage(self): + assert isinstance(self.storage, PgStorage) + + + def test_store_vector(self): + + vector_output = ComplexOutput('vector', 'Vector output', + supported_formats=[FORMATS.GML]) + vector_output.file = get_vector_file() + vector_output.output_format = FORMATS.GML + store_vector = self.storage.store(vector_output) + + assert len(store_vector) == 3 + assert store_vector[0] == STORE_TYPE.DB + assert isinstance(store_vector[1], str) + assert isinstance(store_vector[2], str) + + # Parse reference into dbname, schema and table + reference = store_vector[2].split(".") + + db_url = "postgresql://{}:{}@{}:{}/{}".format( + reference[0], self.password, self.host, self.port, self.user + ) + engine = create_engine(db_url) + # check if table exists + ins = inspect(engine) + assert (reference[2] in ins.get_table_names(schema=reference[1])) + + + def test_store_raster(self): + raster_output = ComplexOutput('raster', 'Raster output', + supported_formats=[FORMATS.GEOTIFF]) + raster_output.file = get_raster_file() + raster_output.output_format = FORMATS.GEOTIFF + + store_raster = self.storage.store(raster_output) + + assert len(store_raster) == 3 + assert store_raster[0] == STORE_TYPE.DB + assert isinstance(store_raster[1], str) + assert isinstance(store_raster[2], str) + + # Parse reference into dbname, schema and table + reference = store_raster[2].split(".") + + db_url = "postgresql://{}:{}@{}:{}/{}".format( + reference[0], self.password, self.host, self.port, self.user + ) + engine = create_engine(db_url) + # check if table exists + ins = inspect(engine) + assert (reference[2] in ins.get_table_names(schema=reference[1])) + + + def test_store_other(self): + text_output = ComplexOutput('txt', 'Plain text output', + supported_formats=[FORMATS.TEXT]) + text_output.file = get_text_file() + text_output.output_format = FORMATS.TEXT + + store_text = self.storage.store(text_output) + + assert len(store_text) == 3 + assert store_text[0] == STORE_TYPE.DB + assert isinstance(store_text[1], str) + assert isinstance(store_text[2], str) + + # Parse reference into dbname, schema and table + reference = store_text[2].split(".") + + db_url = "postgresql://{}:{}@{}:{}/{}".format( + reference[0], self.password, self.host, self.port, self.user + ) + engine = create_engine(db_url) + # check if table exists + ins = inspect(engine) + assert (reference[2] in ins.get_table_names(schema=reference[1])) + + + csv_output = ComplexOutput('csv', 'CSV output', + supported_formats=[FORMATS.CSV]) + csv_output.file = get_csv_file() + csv_output.output_format = FORMATS.CSV + + store_csv = self.storage.store(csv_output) + + assert len(store_csv) == 3 + assert store_csv[0] == STORE_TYPE.DB + assert isinstance(store_csv[1], str) + assert isinstance(store_csv[2], str) + + # Parse reference into dbname, schema and table + reference = store_csv[2].split(".") + + db_url = "postgresql://{}:{}@{}:{}/{}".format( + reference[0], self.password, self.host, self.port, self.user + ) + engine = create_engine(db_url) + # check if table exists + ins = inspect(engine) + assert (reference[2] in ins.get_table_names(schema=reference[1])) + + +class SQLiteStorageTestCase(unittest.TestCase): + """SQLiteStorage test + """ + + def setUp(self): + global TEMP_DIRS + tmp_dir = tempfile.mkdtemp() + TEMP_DIRS.append(tmp_dir) + + self.storage = SQLiteStorage() + self.storage.target = tempfile.mktemp(suffix='.sqlite', prefix='pywpsdb-') + + + def tearDown(self): + # Delete temp file if exists + try: + os.remove(self.storage.target) + except: + pass + + def test_sqlite_storage(self): + assert isinstance(self.storage, SQLiteStorage) + + + def test_store_vector(self): + vector_output = ComplexOutput('vector', 'Vector output', + supported_formats=[FORMATS.GML]) + vector_output.file = get_vector_file() + vector_output.output_format = FORMATS.GML + store_vector = self.storage.store(vector_output) + + assert len(store_vector) == 3 + assert store_vector[0] == STORE_TYPE.DB + assert isinstance(store_vector[1], str) + assert isinstance(store_vector[2], str) + + # Parse reference into path to db and table + reference = store_vector[2].rsplit(".", 1) + + db_url = "sqlite:///{}".format(reference[0]) + engine = create_engine(db_url) + # check if table exists + ins = inspect(engine) + assert (reference[1] in ins.get_table_names()) + + + def test_store_raster(self): + raster_output = ComplexOutput('raster', 'Raster output', + supported_formats=[FORMATS.GEOTIFF]) + raster_output.file = get_raster_file() + raster_output.output_format = FORMATS.GEOTIFF + + store_raster = self.storage.store(raster_output) + + assert len(store_raster) == 3 + assert store_raster[0] == STORE_TYPE.DB + assert isinstance(store_raster[1], str) + assert isinstance(store_raster[2], str) + + # Parse reference into path to db and table + reference = store_raster[2].rsplit(".", 1) + + db_url = "sqlite:///{}".format(reference[0]) + engine = create_engine(db_url) + # check if table exists + ins = inspect(engine) + + assert (reference[1] + "_rasters") in ins.get_table_names() + + + def test_store_other(self): + + # Test text output + text_output = ComplexOutput('txt', 'Plain text output', + supported_formats=[FORMATS.TEXT]) + text_output.file = get_text_file() + text_output.output_format = FORMATS.TEXT + + store_text = self.storage.store(text_output) + + assert len(store_text) == 3 + assert store_text[0] == STORE_TYPE.DB + assert isinstance(store_text[1], str) + assert isinstance(store_text[2], str) + + # Parse reference into path to db and table + reference = store_text[2].rsplit(".", 1) + + db_url = "sqlite:///{}".format(reference[0]) + engine = create_engine(db_url) + # check if table exists + ins = inspect(engine) + assert (reference[1] in ins.get_table_names()) + + # Test CSV output + csv_output = ComplexOutput('csv', 'CSV output', + supported_formats=[FORMATS.CSV]) + csv_output.file = get_csv_file() + csv_output.output_format = FORMATS.CSV + + store_csv = self.storage.store(csv_output) + + assert len(store_csv) == 3 + assert store_csv[0] == STORE_TYPE.DB + assert isinstance(store_csv[1], str) + assert isinstance(store_csv[2], str) + + # Parse reference into path to db and table + reference = store_csv[2].rsplit(".", 1) + + db_url = "sqlite:///{}".format(reference[0]) + + engine = create_engine(db_url) + # check if table exists + ins = inspect(engine) + assert (reference[1] in ins.get_table_names()) +