Edit README and add examples

Author: BetaHuhn

README.md

@@ -1 +1,200 @@
-# metadata-scraper
+<div align="center">
+  
+# metadata-scraper
+
+[![GitHub](https://img.shields.io/github/license/mashape/apistatus.svg)](https://github.com/BetaHuhn/metadata-scraper/blob/master/LICENSE) ![David](https://img.shields.io/david/betahuhn/metadata-scraper) [![npm](https://img.shields.io/npm/v/metadata-scraper)](https://www.npmjs.com/package/metadata-scraper)
+
+A Javascript library for scraping/parsing metadata from a web page.
+
+</div>
+
+## 👋 Introduction
+
+[metadata-scraper](https://github.com/BetaHuhn/metadata-scraper) is a Javascript library which scrapes/parses metadata from web pages. You only need to supply it with a URL or an HTML string and it will use different rules to find the most relevant metadata like:
+
+- Title
+- Description
+- Favicons/Images
+- Language
+- Keywords
+- Author
+- and more (full list [below](#))
+
+
+## 🚀 Get started
+
+Install [metadata-scraper](https://github.com/BetaHuhn/metadata-scraper) via npm:
+```shell
+npm install metadata-scraper
+```
+
+## 📚 Usage
+
+Import `metadata-scraper` and pass it an URL or options object:
+
+```js
+const getMetaData = require('metadata-scraper')
+
+const url = 'https://github.com/BetaHuhn/metadata-scraper'
+
+getMetaData(url).then((data) => {
+	console.log(data)
+})
+```
+
+Or with `async`/`await`:
+
+```js
+const getMetaData = require('metadata-scraper')
+
+async function run() {
+	const url = 'https://github.com/BetaHuhn/metadata-scraper'
+	const data = await getMetaData(url)
+	console.log(data)
+}
+
+run()
+```
+
+This will return:
+
+```js
+{
+  title: 'BetaHuhn/metadata-scraper',
+  description: 'A Javascript library for scraping/parsing metadata from a web page.',
+  language: 'en',
+  url: 'https://github.com/BetaHuhn/metadata-scraper',
+  provider: 'GitHub',
+  twitter: '@github',
+  image: 'https://avatars1.githubusercontent.com/u/51766171?s=400&v=4',
+  icon: 'https://github.githubassets.com/favicons/favicon.svg'
+}
+```
+
+## ⚙️ Configuration
+
+You can change the behaviour of [metadata-scraper](https://github.com/BetaHuhn/metadata-scraper) by passing an options object:
+
+```js
+const getMetaData = require('../lib')
+
+const options = {
+  url: 'https://github.com/BetaHuhn/metadata-scraper', // URL of web page
+	maxRedirects: 0, // Maximum number of redirects to follow (default: 5)
+	ua: 'MyApp', // User-Agent header
+	timeout: 1000, // Request timeout in milliseconds (default: 10000ms)
+	forceImageHttps: false, // Force all image URLs to use https (default: true)
+	customRules: {} // more info below
+}
+
+getMetaData(options).then((data) => {
+	console.log(data)
+})
+```
+
+You can specify the URL by either passing it as the first parameter, or by setting it in the options object.
+
+## 📖 Examples
+
+Here are some examples on how to use [metadata-scraper](https://github.com/BetaHuhn/metadata-scraper):
+
+### Basic
+
+Pass an URL as the first parameter and [metadata-scraper](https://github.com/BetaHuhn/metadata-scraper) automatically scrapes it and returns everything it finds:
+
+```js
+const url = 'https://github.com/BetaHuhn/metadata-scraper'
+const data = await getMetaData(url)
+```
+
+Example file located at [examples/basic.js](/examples/basic.js).
+
+---
+
+### HTML String
+
+If you already have an HTML string and don't want [metadata-scraper](https://github.com/BetaHuhn/metadata-scraper) to make an http request, specify it in the options object:
+
+```js
+const html = `
+  <meta name="og:title" content="Example">
+  <meta name="og:description" content="This is an example.">
+`
+
+const options {
+  html: html, 
+  url: 'https://example.com' // Optional URL to make relative image paths absolute
+}
+
+const data = await getMetaData(options)
+```
+
+Example file located at [examples/html.js](/examples/html.js).
+
+---
+
+### Custom Rules
+
+Look at the `rules.ts` file in the `src` directory to see all rules which will be used.
+
+You can expand [metadata-scraper](https://github.com/BetaHuhn/metadata-scraper) easily by specifying custom rules:
+
+```js
+const options = {
+  url: 'https://github.com/BetaHuhn/metadata-scraper',
+	customRules: {
+		name: {
+			rules: [
+				[ 'meta[name="customName"][content]', (element) => element.getAttribute('content') ]
+			],
+			processor: (text) => text.toLowerCase()
+		}
+	}
+}
+
+const data = await getMetaData(options)
+```
+
+`customRules` needs to contain one or more objects, where the key (name above) is the key which later gets returned. You can then specify different rules for that item in the rules array.
+
+The first item is the query which gets inserted into the browsers querySelector function, and the second item is a function which gets passed the HTML element:
+
+```js
+[ 'querySelector', (element) => element.innerText ]
+```
+
+You can also specify a `processor` function which will process/transform the result of one of the matched rules:
+
+```js
+{
+  processor: (text) => text.toLowerCase()
+}
+```
+
+If you find a useful metatag/rule, let me know and I will add them (or create a PR yourself).
+
+Example file located at [examples/custom.js](/examples/custom.js).
+
+## 💻 Development
+
+Issues and PRs are very welcome!
+
+Please check out the [contributing guide](CONTRIBUTING.md) before you start.
+
+This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). To see differences with previous versions refer to the [CHANGELOG](CHANGELOG.md).
+
+## ❔ About
+
+This library was developed by me ([@betahuhn](https://github.com/BetaHuhn)) in my free time. If you want to support me:
+
+[![Donate via PayPal](https://img.shields.io/badge/paypal-donate-009cde.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=394RTSBEEEFEE)
+
+### Credits
+
+The loader is based on [file-loader](https://github.com/webpack-contrib/file-loader).
+
+## License
+
+Copyright 2020 Maximilian Schiller
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

examples/basic.js

@@ -1,7 +1,7 @@
 const getMetaData = require('../lib')
 
 const run = async function() {
-	const url = 'https://www.sueddeutsche.de/politik/usa-joe-biden-ron-klain-1.5113555'
+	const url = 'https://github.com/BetaHuhn/metadata-scraper'
 	const data = await getMetaData(url)
 	console.log(data)
 }

examples/custom.js

@@ -0,0 +1,19 @@
+const getMetaData = require('../lib')
+
+const options = {
+	customRules: {
+		title: {
+			rules: [
+				[ 'meta[property="customTitle"][content]', (element) => element.getAttribute('content') ]
+			],
+			processor: (text) => text.toLowerCase()
+		}
+	}
+}
+
+const run = async function() {
+	const data = await getMetaData('https://github.com/BetaHuhn/metadata-scraper', options)
+	console.log(data)
+}
+
+run()

examples/html.js

@@ -5,7 +5,8 @@ const run = async function() {
 		<meta name="og:title" content="Example">
 		<meta name="og:description" content="This is an example.">
 	`
-	const data = await getMetaData(html, { html: true, url: 'https://example.com' })
+
+	const data = await getMetaData({ html: html, url: 'https://example.com' })
 	console.log(data)
 }
 

examples/options.js

@@ -1,23 +1,16 @@
 const getMetaData = require('../lib')
 
 const options = {
+	url: 'https://github.com/BetaHuhn/metadata-scraper',
 	maxRedirects: 0, // default: 5
-	ua: 'MyApp', // default: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'
+	ua: 'MyApp',
 	timeout: 1000, // default: 10000
 	forceImageHttps: false, // default: true
-	customRules: {
-		title: {
-			rules: [
-				[ 'meta[property="customTitle"][content]', (element) => element.getAttribute('content') ]
-			],
-			processor: (text) => text.toLowerCase()
-		}
-	}
+	customRules: {}
 }
 
 const run = async function() {
-	const url = 'https://www.sueddeutsche.de/politik/usa-joe-biden-ron-klain-1.5113555'
-	const data = await getMetaData(url, options)
+	const data = await getMetaData(options)
 	console.log(data)
 }
 

package.json

@@ -1,7 +1,7 @@
 {
   "name": "metadata-scraper",
   "version": "0.1.0",
-  "description": "",
+  "description": "A Javascript library for scraping/parsing metadata from a web page.  ",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "scripts": {

src/index.ts

@@ -55,7 +55,16 @@ const runRule = function(ruleSet: RuleSet, doc: Document, context: Context) {
 	return undefined
 }
 
-const getMetaData = async function(url: string, inputOptions: Partial<Options> = {}) {
+const getMetaData = async function(input: string | Partial<Options>, inputOptions: Partial<Options> = {}) {
+
+	let url
+	if (typeof input === 'object') {
+		inputOptions = input
+		url = input.url || ''
+	} else {
+		url = input
+	}
+
 	const options = Object.assign({}, defaultOptions, inputOptions)
 
 	const rules: Record<string, RuleSet> = { ...metaDataRules }
@@ -78,8 +87,7 @@ const getMetaData = async function(url: string, inputOptions: Partial<Options> =
 		})
 		html = response.body
 	} else {
-		html = url
-		url = options.url || ''
+		html = options.html
 	}
 
 	const metadata: MetaData = {}

src/types.ts

@@ -33,7 +33,7 @@ export interface Options {
     ua?: string
     timeout?: number
     forceImageHttps?: boolean
-    html?: boolean
+    html?: string
     url?: string
     customRules?: Record<string, RuleSet>
 }