github impoort

Author: Pedro

LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright Pedro (c) 2014 
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

README.md

@@ -0,0 +1,55 @@
+# tq
+
+tq is command line utility  that performs performs an HTML element selection on HTML content passed to the stdin. Using css selectors that everybody knows.
+
+Since input comes from stdin and output is sent to stout. It can easily be be used inside traditional UNIX pipelines to extract content from webpages and html files.
+
+tq provides extra formating options such as json-encoding or newlines squashing, so it can pley nicely with everyones favourite command line tooling.
+
+
+## Instalation
+
+	sudo pip install https://github.com/plainas/tq/zipball/stable
+
+
+## Example usage
+
+Get headlines from hacker news
+
+	curl https://news.ycombinator.com/news | tq -tj ".title a"
+
+Get the title of an html document stored in a file
+
+	cat mydocument.html | tq -t title
+
+Get all the images from a webpage
+
+	TODO: add this example
+
+
+Notice that tq doesn't provide a way to make http requests or read files. You can use your favorite HTTP client, or provide the html source from any source you want.
+
+For a modern, user friendly http client, check httpie. Or you can just use curl, wget, netcat, etc.
+
+## Command options
+
+  * `selector`
+    A css selector
+
+  * `-t, --text`
+    Outputs only the inner text of the selected elements.
+  
+  * `-q, --squash`
+    Squash lines.
+  
+  * `-s, --squash-space`
+    Squash spaces.
+  
+  * `-j, --json-lines`
+    JSON encode each match.
+  
+  * `-J, --json`
+    Output as json array of strings.
+  
+  * `-v, --version`
+    Prints tq version

bin/tq

@@ -0,0 +1,5 @@
+#!/usr/bin/python3
+import tq
+
+if __name__ == '__main__':
+    tq.main()

doc/compile_manpage.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# ronn is used to turn the markdown into a manpage.
+# Get ronn at https://github.com/rtomayko/ronn
+# Alternately, since ronn is a Ruby gem, you can just
+# `gem install ronn`
+
+ronn --roff tq.1.md

doc/tq.1

@@ -0,0 +1,55 @@
+.\" generated with Ronn/v0.7.3
+.\" http://github.com/rtomayko/ronn/tree/0.7.3
+.
+.TH "TQ" "1" "September 2015" "" ""
+.
+.SH "NAME"
+\fBtq\fR \- Terminal based HTML query tool
+.
+.SH "SYNOPSIS"
+cat file\.html | \fBtq\fR [\fIoptions\fR] SELECTOR
+.
+.SH "DESCRIPTION"
+Perform a css query with SELECTOR on an html document passed to the standard input\.
+.
+.SH "OPTIONS"
+.
+.IP "\(bu" 4
+\fBselector\fR A css selector
+.
+.IP "\(bu" 4
+\fB\-t, \-\-text\fR Outputs only the inner text of the selected elements\.
+.
+.IP "\(bu" 4
+\fB\-q, \-\-squash\fR Squash lines\.
+.
+.IP "\(bu" 4
+\fB\-s, \-\-squash\-space\fR Squash spaces\.
+.
+.IP "\(bu" 4
+\fB\-j, \-\-json\-lines\fR JSON encode each match\.
+.
+.IP "\(bu" 4
+\fB\-J, \-\-json\fR Output as json array of strings\.
+.
+.IP "\(bu" 4
+\fB\-v, \-\-version\fR Prints tq version
+.
+.IP "" 0
+.
+.SH "EXAMPLES"
+.
+.SS "Get headlines from hacker news"
+curl https://news\.ycombinator\.com/news | tq \-tj "\.title a"
+.
+.SS "Download a gallery of nice forest pictures from flickr"
+curl \-s \'https://www\.flickr\.com/photos/tgerus/galleries/72157622468645106/\' | tq "\.gallery\-photos img"
+.
+.SH "AUTHORS"
+\fBtq\fR was written by Pedro \fIpedro@example\.com\fR\.
+.
+.SH "DISTRIBUTION"
+The latest version of tq may be downloaded from https://github\.com/plainas/tq
+.
+.SH "SEE ALSO"
+curl(1), wget(1), jq(1)

doc/tq.1.md

@@ -0,0 +1,57 @@
+tq(1) -- Terminal based HTML query tool
+=============================================
+
+## SYNOPSIS
+
+cat file.html | `tq` [<options>] SELECTOR
+
+## DESCRIPTION
+
+Perform a css query with SELECTOR on an html document passed to the standard input.
+
+## OPTIONS
+
+  * `selector`
+    A css selector
+
+  * `-t, --text`
+    Outputs only the inner text of the selected elements.
+  
+  * `-q, --squash`
+    Squash lines.
+  
+  * `-s, --squash-space`
+    Squash spaces.
+  
+  * `-j, --json-lines`
+    JSON encode each match.
+  
+  * `-J, --json`
+    Output as json array of strings.
+  
+  * `-v, --version`
+    Prints tq version
+
+
+## EXAMPLES
+
+
+### Get headlines from hacker news
+
+curl https://news.ycombinator.com/news | tq -tj ".title a"
+
+### Download a gallery of nice forest pictures from flickr
+
+curl -s 'https://www.flickr.com/photos/tgerus/galleries/72157622468645106/' | tq  ".gallery-photos img"
+
+
+## AUTHORS
+
+`tq` was written by Pedro <[email protected]>.
+
+## DISTRIBUTION
+The latest version of tq may be downloaded from https://github.com/plainas/tq
+
+## SEE ALSO
+
+curl(1), wget(1), jq(1)

setup.py

@@ -0,0 +1,15 @@
+#!/usr/bin/python3
+
+from distutils.core import setup
+
+setup(
+    name='tq',
+    version='0.1',
+    description='comand line css selector',
+    author='Pedro',
+    author_email='[email protected]',
+    url='https://github.com/plainas/tq',
+    packages= ['tq'],
+    scripts=['bin/tq'],
+    install_requires=["beautifulsoup4=4.4.0"]
+)

tq/__init__.py

@@ -0,0 +1,88 @@
+"""
+Test non unicode input with:
+curl https://www.flashback.org/| ./tq.py -Jt ".td_forum"
+
+Test unicode input
+curl https://news.ycombinator.com/news| ./tq.py -Jt ".title a"
+
+curl https://www.flashback.org/t2494391| ./tq.py -j ".post_message"
+
+"""
+
+#TODO: use add_mutually_exclusive_group()
+#TODO: help2man proved ineficient. Copy and import this script and use it. Instructions are in the source, pretty straight forward
+
+#https://github.com/pwman3/pwman3/blob/d718a01fa8038893e42416b59cdfcda3935fe878/build_manpage.py
+
+
+import sys
+from bs4 import BeautifulSoup
+import argparse
+import json
+import codecs
+import io
+
+version = "0.0.1"
+
+#parser = argparse.ArgumentParser(description="Performs a css selection on an HTML document.", prog= "TQ", usage='curl url | tq [options]')
+parser = argparse.ArgumentParser(description="Performs a css selection on an HTML document.", prog= "tq")
+parser.add_argument("selector", help="A css selector")
+parser.add_argument("-t", "--text",			action="store_true", help="Outputs only the inner text of the selected elements.")
+parser.add_argument("-q", "--squash",		action="store_true", help="Squash lines.")
+parser.add_argument("-s", "--squash-space",	action="store_true", help="Squash spaces.")
+parser.add_argument("-j", "--json-lines",	action="store_true", help="JSON encode each match.")
+parser.add_argument("-J", "--json",			action="store_true", help="Output as json array of strings.")
+parser.add_argument("-v", "--version",		action="store_true", help=version)
+
+args = parser.parse_args()
+
+
+def get_parser(formatter_class=argparse.HelpFormatter):
+    """
+    this is here just to be picked up by build_manpage
+    """
+    return parser
+
+
+
+def main():
+
+    if args.version:
+    	print(version)
+    	system.exit()
+
+    if not args.selector:
+        system.exit("ERROR! No selector")
+
+    if args.json and args.json_lines:
+        sys.exit("ERROR! --json and --json-lines options cannot be used simultaniously")
+
+
+    def get_els(css_selector):
+        #input_stream = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8', errors='ignore')
+        input_stream = io.TextIOWrapper(sys.stdin.buffer, errors='ignore')
+        soup = BeautifulSoup(input_stream, "html.parser")
+        return soup.select(css_selector)
+
+
+    selected_els = get_els(args.selector)
+
+    if args.text:
+        selected_els = [el.get_text() for el in selected_els]
+
+    if args.squash:
+        selected_els = [el.replace('\n', ' ').el('\r', '') for el in selected_els]
+    	
+    if args.squash_space:
+        selected_els = [' '.join( el.split(' ') ) for el in selected_els]
+
+    if args.json or args.json_lines:
+        selected_els = [json.dumps(str(el_text)) for el_text in selected_els]
+
+
+    if args.json:
+        sys.stdout.write(json.dumps(selected_els, indent=1))
+        sys.stdout.write("\n")
+    else:
+        for el_text in selected_els:
+            sys.stdout.write(str(el_text) + "\n")