Web Scraper API

The Web Scraper API provides web scraper services to retrieve HTML and plaintext data from any website URL.

/v1/webscraper

`HTTP GET`

Returns the HTML or plaintext data scraped from a given URL. Maximum size of data returned is 2MB.

Parameters

url (required) - URL to scrape.

text_only (optional) - whether to only extract visible text (ignores HTML tags and metadata). Must be either true or false. Default is false.

user_agent (optional) - User-Agent string to use in the request header.

Headers

X-Api-Key (required) - API Key associated with your account.

Sample Request URL

Live Demo!

https://api.api-ninjas.com/v1/webscraper?url=

Sample Response

{
  "data": "<!doctype html>
<html>
<head>
    <title>Example Domain</title>

    <meta charset="utf-8" />
    <meta http-equiv="Content-type" content="text/html; charset=utf-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1" />
    <style type="text/css">
    body {
        background-color: #f0f0f2;
        margin: 0;
        padding: 0;
        font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
        
    }
    div {
        width: 600px;
        margin: 5em auto;
        padding: 2em;
        background-color: #fdfdff;
        border-radius: 0.5em;
        box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02);
    }
    a:link, a:visited {
        color: #38488f;
        text-decoration: none;
    }
    @media (max-width: 700px) {
        div {
            margin: 0 auto;
            width: auto;
        }
    }
    </style>    
</head>

<body>
<div>
    <h1>Example Domain</h1>
    <p>This domain is for use in illustrative examples in documents. You may use this
    domain in literature without prior coordination or asking for permission.</p>
    <p><a href="https://www.iana.org/domains/example">More information...</a></p>
</div>
</body>
</html>
"
}

Code Examples

import requests

api_url = 'https://api.api-ninjas.com/v1/webscraper?url=https://example.com'
response = requests.get(api_url, headers={'X-Api-Key': 'YOUR_API_KEY'})
if response.status_code == requests.codes.ok:
    print(response.text)
else:
    print("Error:", response.status_code, response.text)

$.ajax({
    method: 'GET',
    url: 'https://api.api-ninjas.com/v1/webscraper?url=https://example.com',
    headers: { 'X-Api-Key': 'YOUR_API_KEY'},
    contentType: 'application/json',
    success: function(result) {
        console.log(result);
    },
    error: function ajaxError(jqXHR) {
        console.error('Error: ', jqXHR.responseText);
    }
});

const request = require('request');
request.get({
  url: 'https://api.api-ninjas.com/v1/webscraper?url=https://example.com',
  headers: {
    'X-Api-Key': 'YOUR_API_KEY'
  },
}, function(error, response, body) {
  if(error) return console.error('Request failed:', error);
  else if(response.statusCode != 200) return console.error('Error:', response.statusCode, body.toString('utf8'));
  else console.log(body)
});

import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;

import com.fasterxml.jackson.databind.ObjectMapper;

URL url = new URL("https://api.api-ninjas.com/v1/webscraper?url=https://example.com");
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setRequestProperty("accept", "application/json");
InputStream responseStream = connection.getInputStream();
ObjectMapper mapper = new ObjectMapper();
JsonNode root = mapper.readTree(responseStream);
System.out.println(root.path("fact").asText());

let url = URL(string: "https://api.api-ninjas.com/v1/webscraper?url=https://example.com")!
var request = URLRequest(url: url)
request.setValue("YOUR_API_KEY", forHTTPHeaderField: "X-Api-Key")
let task = URLSession.shared.dataTask(with: request) {(data, response, error) in
    guard let data = data else { return }
    print(String(data: data, encoding: .utf8)!)
}
task.resume()

If your programming language is not listed in the Code Example above, you can still make API calls by using a HTTP request library written in your programming language and following the above documentation.