diff --git a/.gitignore b/.gitignore index eca7a0a..b719ddb 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,6 @@ shodan.egg-info/* tmp/* MANIFEST .vscode/ -PKG-INFO \ No newline at end of file +PKG-INFO +venv/* +.idea/* \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index adce430..8d9f9aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,129 @@ CHANGELOG ========= +1.28.0 +------ +* Add the ability to whitelist a specific vulnerability in Shodan Monitor instead of whitelisting the while IP:port +* Show scan ID when scanning without showing results (credit to @seadog007) +* Handle bad gateway errors (credit to @yaron-cider) + + +1.27.0 +------ +* New command: ``shodan alert export`` to save the current network monitoring configuration +* New command: ``shodan alert import`` to restore a previous network monitoring configuration +* Automatically rate limit API requests to 1 request per second (credit to @malvidin) + +1.26.1 +------ +* Fix a unicode issue that caused the streams to get truncated and error out due to invalid JSON + +1.26.0 +------ +* Add the ability to create custom data streams in the Shodan() class as well as the CLI (``shodan stream --custom-filters ``) + +1.25.0 +------ +* Add new CLI command: shodan alert download + +1.24.0 +------ +* Add new CLI command: shodan alert stats + +1.23.0 +------ +* Add new CLI command: shodan alert domain + +1.22.1 +------ +* Fix bug when converting data file to CSV using Python3 + +1.22.0 +------ +* Add support for new vulnerability streaming endpoints + +1.21.3 +------ +* Fix geo.json file converter + +1.21.2 +------ +* Add support for paging through the domain information + +1.21.1 +------ +* Add ``history`` and ``type`` parameters to ``Shodan.dns.domain_info()`` method and CLI command + +1.21.0 +------ +* New API methods ``api.search_facets()`` and ``api.search_filters()`` to get a list of available facets and filters. + +1.20.0 +------ +* New option "-S" for **shodan domain** to save results from the lookup +* New option "-D" for **shodan domain** to lookup open ports for IPs in the results + +1.19.0 +------ +* New method to edit the list of IPs for an existing network alert + +1.18.0 +------ +* Add library methods for the new Notifications API + +1.17.0 +------ +* Fix bug that caused unicode error when printing domain information (#106) +* Add flag to let users get their IPv6 address **shodan myip -6**(#35) + +1.16.0 +------ +* Ability to specify list of fields to include when converting to CSV/ Excel (#107) +* Filter the Shodan Firehose based on tags in the banner + +1.15.0 +------ +* New option "--skip" for download command to help users resume a download + +1.14.0 +------ +* New command **shodan version** (#104). +* Only change api_key file permissions if needed (#103) + +1.13.0 +------ +* New command **shodan domain** to lookup a domain in Shodan's DNS database +* Override environment configured settings if explicit proxy settings are supplied (@cudeso) + +1.12.1 +------ +* Fix Excel file conversion that resulted in empty .xlsx files + +1.12.0 +------ +* Add new methods to ignore/ unignore trigger notifications + +1.11.1 +------ +* Allow a single network alert to monitor multiple IP ranges (#93) + +1.11.0 +------ +* New command **shodan scan list** to list recently launched scans +* New command **shodan alert triggers** to list the available notification triggers +* New command **shodan alert enable** to enable a notification trigger +* New command **shodan alert disable** to disable a notification trigger +* New command **shodan alert info** to show details of a specific alert +* Include timestamp, vulns and tags in CSV converter (#85) +* Fixed bug that caused an exception when parsing uncompressed data files in Python3 +* Code quality improvements +* Thank you for contributions from @wagner-certat, @cclauss, @opt9, @voldmar and Antoine Neuenschwander + +1.10.4 +------ +* Fix a bug when showing old banner records that don't have the "transport" property +* Code quality improvements (bare excepts) + 1.10.3 ------ * Change bare 'except:' statements to 'except Exception:' or more specific ones diff --git a/README.rst b/README.rst index d0b1f10..14f6717 100644 --- a/README.rst +++ b/README.rst @@ -18,8 +18,10 @@ Features - `Fast/ bulk IP lookups `_ - Streaming API support for real-time consumption of Shodan firehose - `Network alerts (aka private firehose) `_ +- `Manage Email Notifications `_ - Exploit search API fully implemented - Bulk data downloads +- Access the Shodan DNS DB to view domain information - `Command-line interface `_ .. image:: https://cli.shodan.io/img/shodan-cli-preview.png diff --git a/docs/examples/cert-stream.rst b/docs/examples/cert-stream.rst index e3e72c1..b01440e 100644 --- a/docs/examples/cert-stream.rst +++ b/docs/examples/cert-stream.rst @@ -24,7 +24,6 @@ information. # information. # # Author: achillean - import shodan import sys @@ -35,7 +34,7 @@ information. # Setup the api api = shodan.Shodan(API_KEY) - print 'Listening for certs...' + print('Listening for certs...') for banner in api.stream.ports([443, 8443]): if 'ssl' in banner: # Print out all the SSL information that Shodan has collected diff --git a/docs/examples/gifcreator.rst b/docs/examples/gifcreator.rst index ad9efc8..e4a43c3 100644 --- a/docs/examples/gifcreator.rst +++ b/docs/examples/gifcreator.rst @@ -106,7 +106,7 @@ There are a few key Shodan methods/ parameters that make the script work: os.system('rm -f /tmp/gif-image-*.jpg') # Show a progress indicator - print result['ip_str'] + print(result['ip_str']) -The full code is also available on GitHub: https://gist.github.com/achillean/963eea552233d9550101 \ No newline at end of file +The full code is also available on GitHub: https://gist.github.com/achillean/963eea552233d9550101 diff --git a/docs/examples/query-summary.rst b/docs/examples/query-summary.rst index 7a60716..66e15fe 100644 --- a/docs/examples/query-summary.rst +++ b/docs/examples/query-summary.rst @@ -48,7 +48,7 @@ and country. # Input validation if len(sys.argv) == 1: - print 'Usage: %s ' % sys.argv[0] + print('Usage: %s ' % sys.argv[0]) sys.exit(1) try: @@ -62,22 +62,22 @@ and country. # And it also runs faster than doing a search(). result = api.count(query, facets=FACETS) - print 'Shodan Summary Information' - print 'Query: %s' % query - print 'Total Results: %s\n' % result['total'] + print('Shodan Summary Information') + print('Query: %s' % query) + print('Total Results: %s\n' % result['total']) # Print the summary info from the facets for facet in result['facets']: - print FACET_TITLES[facet] + print(FACET_TITLES[facet]) for term in result['facets'][facet]: - print '%s: %s' % (term['value'], term['count']) + print('%s: %s' % (term['value'], term['count'])) # Print an empty line between summary info - print '' + print('') - except Exception, e: - print 'Error: %s' % e + except Exception as e: + print('Error: %s' % e) sys.exit(1) """ diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 00c4d34..62744af 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -56,7 +56,7 @@ Now that we have our API object all good to go, we're ready to perform a search: print('IP: {}'.format(result['ip_str'])) print(result['data']) print('') - except shodan.APIError, e: + except shodan.APIError as e: print('Error: {}'.format(e)) Stepping through the code, we first call the :py:func:`Shodan.search` method on the `api` object which diff --git a/requirements.txt b/requirements.txt index 4fa2ed6..2692414 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,6 @@ click click-plugins colorama requests>=2.2.1 -XlsxWriter \ No newline at end of file +XlsxWriter +ipaddress;python_version<='2.7' +tldextract \ No newline at end of file diff --git a/setup.py b/setup.py index ef991f1..53bbd9a 100755 --- a/setup.py +++ b/setup.py @@ -2,23 +2,25 @@ from setuptools import setup + DEPENDENCIES = open('requirements.txt', 'r').read().split('\n') README = open('README.rst', 'r').read() + setup( - name = 'shodan', - version = '1.10.4', - description = 'Python library and command-line utility for Shodan (https://developer.shodan.io)', - long_description = README, - long_description_content_type = 'text/x-rst', - author = 'John Matherly', - author_email = 'jmath@shodan.io', - url = 'http://github.com/achillean/shodan-python/tree/master', - packages = ['shodan', 'shodan.cli', 'shodan.cli.converter'], - entry_points = {'console_scripts': ['shodan = shodan.__main__:main']}, - install_requires = DEPENDENCIES, - keywords = ['security', 'network'], - classifiers = [ + name='shodan', + version='1.31.0', + description='Python library and command-line utility for Shodan (https://developer.shodan.io)', + long_description=README, + long_description_content_type='text/x-rst', + author='John Matherly', + author_email='jmath@shodan.io', + url='https://github.com/achillean/shodan-python', + packages=['shodan', 'shodan.cli', 'shodan.cli.converter'], + entry_points={'console_scripts': ['shodan=shodan.__main__:main']}, + install_requires=DEPENDENCIES, + keywords=['security', 'network'], + classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', diff --git a/shodan/__main__.py b/shodan/__main__.py index 709a010..4093b94 100644 --- a/shodan/__main__.py +++ b/shodan/__main__.py @@ -22,6 +22,7 @@ search stats stream + trends """ @@ -29,11 +30,13 @@ import csv import os import os.path +import pkg_resources import shodan import shodan.helpers as helpers import threading import requests import time +import json # The file converters that are used to go from .json.gz to various other formats from shodan.cli.converter import CsvConverter, KmlConverter, GeoJsonConverter, ExcelConverter, ImagesConverter @@ -49,8 +52,22 @@ from click_plugins import with_plugins from pkg_resources import iter_entry_points +# Large subcommands are stored in separate modules +from shodan.cli.alert import alert +from shodan.cli.data import data +from shodan.cli.organization import org +from shodan.cli.scan import scan + + # Make "-h" work like "--help" CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) +CONVERTERS = { + 'kml': KmlConverter, + 'csv': CsvConverter, + 'geo.json': GeoJsonConverter, + 'images': ImagesConverter, + 'xlsx': ExcelConverter, +} # Define a basestring type if necessary for Python3 compatibility try: @@ -67,34 +84,35 @@ def main(): pass -# Large subcommands are stored in separate modules -from shodan.cli.alert import alert -from shodan.cli.data import data -from shodan.cli.organization import org -from shodan.cli.scan import scan +# Setup the large subcommands main.add_command(alert) main.add_command(data) main.add_command(org) main.add_command(scan) -CONVERTERS = { - 'kml': KmlConverter, - 'csv': CsvConverter, - 'geo.json': GeoJsonConverter, - 'images': ImagesConverter, - 'xlsx': ExcelConverter, -} @main.command() -@click.argument('input', metavar='') +@click.option('--fields', help='List of properties to output.', default=None) +@click.argument('input', metavar='', type=click.Path(exists=True)) @click.argument('format', metavar='', type=click.Choice(CONVERTERS.keys())) -def convert(input, format): +def convert(fields, input, format): """Convert the given input data file into a different format. The following file formats are supported: kml, csv, geo.json, images, xlsx Example: shodan convert data.json.gz kml """ + file_size = 0 + # Check that the converter allows a custom list of fields + converter_class = CONVERTERS.get(format) + if fields: + if not hasattr(converter_class, 'fields'): + raise click.ClickException('File format doesnt support custom list of fields') + converter_class.fields = [item.strip() for item in fields.split(',')] # Use the custom fields the user specified + + # click.Path ensures that file path exists + file_size = os.path.getsize(input) + # Get the basename for the input file basename = input.replace('.json.gz', '').replace('.json', '') @@ -110,9 +128,9 @@ def convert(input, format): progress_bar_thread.start() # Initialize the file converter - converter = CONVERTERS.get(format)(fout) + converter = converter_class(fout) - converter.process([input]) + converter.process([input], file_size) finished_event.set() progress_bar_thread.join() @@ -123,6 +141,73 @@ def convert(input, format): click.echo(click.style('\rSuccessfully created new file: {}'.format(filename), fg='green')) +@main.command(name='domain') +@click.argument('domain', metavar='') +@click.option('--details', '-D', help='Lookup host information for any IPs in the domain results', default=False, is_flag=True) +@click.option('--save', '-S', help='Save the information in the a file named after the domain (append if file exists).', default=False, is_flag=True) +@click.option('--history', '-H', help='Include historical DNS data in the results', default=False, is_flag=True) +@click.option('--type', '-T', help='Only returns DNS records of the provided type', default=None) +def domain_info(domain, details, save, history, type): + """View all available information for a domain""" + key = get_api_key() + api = shodan.Shodan(key) + + try: + info = api.dns.domain_info(domain, history=history, type=type) + except shodan.APIError as e: + raise click.ClickException(e.value) + + # Grab the host information for any IP records that were returned + hosts = {} + if details: + ips = [record['value'] for record in info['data'] if record['type'] in ['A', 'AAAA']] + ips = set(ips) + + fout = None + if save: + filename = u'{}-hosts.json.gz'.format(domain) + fout = helpers.open_file(filename) + + for ip in ips: + try: + hosts[ip] = api.host(ip) + + # Store the banners if requested + if fout: + for banner in hosts[ip]['data']: + if 'placeholder' not in banner: + helpers.write_banner(fout, banner) + except shodan.APIError: + pass # Ignore any API lookup errors as this isn't critical information + + # Save the DNS data + if save: + filename = u'{}.json.gz'.format(domain) + fout = helpers.open_file(filename) + + for record in info['data']: + helpers.write_banner(fout, record) + + click.secho(info['domain'].upper(), fg='green') + + click.echo('') + for record in info['data']: + click.echo( + u'{:32} {:14} {}'.format( + click.style(record['subdomain'], fg='cyan'), + click.style(record['type'], fg='yellow'), + record['value'] + ), + nl=False, + ) + + if record['value'] in hosts: + host = hosts[record['value']] + click.secho(u' Ports: {}'.format(', '.join([str(port) for port in sorted(host['ports'])])), fg='blue', nl=False) + + click.echo('') + + @main.command() @click.argument('key', metavar='') def init(key): @@ -131,7 +216,7 @@ def init(key): shodan_dir = os.path.expanduser(SHODAN_CONFIG_DIR) if not os.path.isdir(shodan_dir): try: - os.mkdir(shodan_dir) + os.makedirs(shodan_dir) except OSError: raise click.ClickException('Unable to create directory to store the Shodan API key ({})'.format(shodan_dir)) @@ -151,6 +236,7 @@ def init(key): os.chmod(keyfile, 0o600) + @main.command() @click.argument('query', metavar='', nargs=-1) def count(query): @@ -175,10 +261,11 @@ def count(query): @main.command() +@click.option('--fields', help='Specify the list of properties to download instead of grabbing the full banner', default=None, type=str) @click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=1000, type=int) @click.argument('filename', metavar='') @click.argument('query', metavar='', nargs=-1) -def download(limit, filename, query): +def download(fields, limit, filename, query): """Download search results and save them in a compressed JSON file.""" key = get_api_key() @@ -197,20 +284,24 @@ def download(limit, filename, query): if not filename.endswith('.json.gz'): filename += '.json.gz' + # Strip out any whitespace in the fields and turn them into an array + if fields is not None: + fields = [item.strip() for item in fields.split(',')] + # Perform the search api = shodan.Shodan(key) try: total = api.count(query)['total'] info = api.info() - except: + except Exception: raise click.ClickException('The Shodan API is unresponsive at the moment, please try again later.') # Print some summary information about the download request - click.echo('Search query:\t\t\t%s' % query) - click.echo('Total number of results:\t%s' % total) - click.echo('Query credits left:\t\t%s' % info['unlocked_left']) - click.echo('Output file:\t\t\t%s' % filename) + click.echo('Search query:\t\t\t{}'.format(query)) + click.echo('Total number of results:\t{}'.format(total)) + click.echo('Query credits left:\t\t{}'.format(info['unlocked_left'])) + click.echo('Output file:\t\t\t{}'.format(filename)) if limit > total: limit = total @@ -222,7 +313,7 @@ def download(limit, filename, query): with helpers.open_file(filename, 'w') as fout: count = 0 try: - cursor = api.search_cursor(query, minify=False) + cursor = api.search_cursor(query, minify=False, fields=fields) with click.progressbar(cursor, length=limit) as bar: for banner in bar: helpers.write_banner(fout, banner) @@ -275,7 +366,6 @@ def host(format, history, filename, save, ip): raise click.ClickException(e.value) - @main.command() def info(): """Shows general information about your account""" @@ -308,7 +398,6 @@ def parse(color, fields, filters, filename, separator, filenames): has_filters = len(filters) > 0 - # Setup the output file handle fout = None if filename: @@ -333,7 +422,7 @@ def parse(color, fields, filters, filename, separator, filenames): helpers.write_banner(fout, banner) # Loop over all the fields and print the banner as a row - for field in fields: + for i, field in enumerate(fields): tmp = u'' value = get_banner_field(banner, field) if value: @@ -351,19 +440,26 @@ def parse(color, fields, filters, filename, separator, filenames): if color: tmp = click.style(tmp, fg=COLORIZE_FIELDS.get(field, 'white')) - # Add the field information to the row - row += tmp - row += separator + # Add the field information to the row + if i > 0: + row += separator + row += tmp click.echo(row) @main.command() -def myip(): +@click.option('--ipv6', '-6', is_flag=True, default=False, help='Try to use IPv6 instead of IPv4') +def myip(ipv6): """Print your external IP address""" key = get_api_key() api = shodan.Shodan(key) + + # Use the IPv6-enabled domain if requested + if ipv6: + api.base_url = 'https://apiv6.shodan.io' + try: click.echo(api.tools.myip()) except shodan.APIError as e: @@ -400,7 +496,7 @@ def search(color, fields, limit, separator, query): # Perform the search api = shodan.Shodan(key) try: - results = api.search(query, limit=limit) + results = api.search(query, limit=limit, minify=False, fields=fields) except shodan.APIError as e: raise click.ClickException(e.value) @@ -519,7 +615,7 @@ def stats(limit, facets, filename, query): if len(values) > counter: has_items = True row[pos] = values[counter]['value'] - row[pos+1] = values[counter]['count'] + row[pos + 1] = values[counter]['count'] pos += 2 @@ -532,20 +628,23 @@ def stats(limit, facets, filename, query): @main.command() -@click.option('--color/--no-color', default=True) +@click.option('--streamer', help='Specify a custom Shodan stream server to use for grabbing data.', default='https://stream.shodan.io', type=str) @click.option('--fields', help='List of properties to output.', default='ip_str,port,hostnames,data') @click.option('--separator', help='The separator between the properties of the search results.', default='\t') -@click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=-1, type=int) @click.option('--datadir', help='Save the stream data into the specified directory as .json.gz files.', default=None, type=str) -@click.option('--ports', help='A comma-separated list of ports to grab data on.', default=None, type=str) -@click.option('--quiet', help='Disable the printing of information to the screen.', is_flag=True) -@click.option('--timeout', help='Timeout. Should the shodan stream cease to send data, then timeout after seconds.', default=0, type=int) -@click.option('--streamer', help='Specify a custom Shodan stream server to use for grabbing data.', default='https://stream.shodan.io', type=str) -@click.option('--countries', help='A comma-separated list of countries to grab data on.', default=None, type=str) @click.option('--asn', help='A comma-separated list of ASNs to grab data on.', default=None, type=str) @click.option('--alert', help='The network alert ID or "all" to subscribe to all network alerts on your account.', default=None, type=str) +@click.option('--countries', help='A comma-separated list of countries to grab data on.', default=None, type=str) +@click.option('--custom-filters', help='A space-separated list of filters query to grab data on.', default=None, type=str) +@click.option('--ports', help='A comma-separated list of ports to grab data on.', default=None, type=str) +@click.option('--tags', help='A comma-separated list of tags to grab data on.', default=None, type=str) +@click.option('--vulns', help='A comma-separated list of vulnerabilities to grab data on.', default=None, type=str) +@click.option('--limit', help='The number of results you want to download. -1 to download all the data possible.', default=-1, type=int) @click.option('--compresslevel', help='The gzip compression level (0-9; 0 = no compression, 9 = most compression', default=9, type=int) -def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, streamer, countries, asn, alert, compresslevel): +@click.option('--timeout', help='Timeout. Should the shodan stream cease to send data, then timeout after seconds.', default=0, type=int) +@click.option('--color/--no-color', default=True) +@click.option('--quiet', help='Disable the printing of information to the screen.', is_flag=True) +def stream(streamer, fields, separator, datadir, asn, alert, countries, custom_filters, ports, tags, vulns, limit, compresslevel, timeout, color, quiet): """Stream data in real-time.""" # Setup the Shodan API key = get_api_key() @@ -571,9 +670,15 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre stream_type.append('asn') if alert: stream_type.append('alert') + if tags: + stream_type.append('tags') + if vulns: + stream_type.append('vulns') + if custom_filters: + stream_type.append('custom_filters') if len(stream_type) > 1: - raise click.ClickException('Please use --ports, --countries OR --asn. You cant subscribe to multiple filtered streams at once.') + raise click.ClickException('Please use --ports, --countries, --custom, --tags, --vulns OR --asn. You cant subscribe to multiple filtered streams at once.') stream_args = None @@ -595,6 +700,15 @@ def stream(color, fields, separator, limit, datadir, ports, quiet, timeout, stre if countries: stream_args = countries.split(',') + if tags: + stream_args = tags.split(',') + + if vulns: + stream_args = vulns.split(',') + + if custom_filters: + stream_args = custom_filters + # Flatten the list of stream types # Possible values are: # - all @@ -613,7 +727,10 @@ def _create_stream(name, args, timeout): 'alert': api.stream.alert(args, timeout=timeout), 'asn': api.stream.asn(args, timeout=timeout), 'countries': api.stream.countries(args, timeout=timeout), + 'custom_filters': api.stream.custom(args, timeout=timeout), 'ports': api.stream.ports(args, timeout=timeout), + 'tags': api.stream.tags(args, timeout=timeout), + 'vulns': api.stream.vulns(args, timeout=timeout), }.get(name, 'all') stream = _create_stream(stream_type, stream_args, timeout=timeout) @@ -641,9 +758,9 @@ def _create_stream(name, args, timeout): if datadir: cur_time = timestr() if cur_time != last_time: - last_time = cur_time - fout.close() - fout = open_streaming_file(datadir, last_time) + last_time = cur_time + fout.close() + fout = open_streaming_file(datadir, last_time) helpers.write_banner(fout, banner) # Print the banner information to stdout @@ -688,6 +805,102 @@ def _create_stream(name, args, timeout): stream = _create_stream(stream_type, stream_args, timeout=timeout) +@main.command() +@click.option('--facets', help='List of facets to get summary information on, if empty then show query total results over time', default='', type=str) +@click.option('--filename', '-O', help='Save the full results in the given file (append if file exists).', default=None) +@click.option('--save', '-S', help='Save the full results in the a file named after the query (append if file exists).', default=False, is_flag=True) +@click.argument('query', metavar='', nargs=-1) +def trends(filename, save, facets, query): + """Search Shodan historical database""" + key = get_api_key() + api = shodan.Shodan(key) + + # Create the query string out of the provided tuple + query = ' '.join(query).strip() + facets = facets.strip() + + # Make sure the user didn't supply an empty query or facets + if query == '': + raise click.ClickException('Empty search query') + + # Convert comma-separated facets string to list + parsed_facets = [] + for facet in facets.split(','): + if not facet: + continue + + parts = facet.strip().split(":") + if len(parts) > 1: + parsed_facets.append((parts[0], parts[1])) + else: + parsed_facets.append((parts[0])) + + # Perform the search + try: + results = api.trends.search(query, facets=parsed_facets) + except shodan.APIError as e: + raise click.ClickException(e.value) + + # Error out if no results were found + if results['total'] == 0: + raise click.ClickException('No search results found') + + result_facets = [] + if results.get("facets"): + result_facets = list(results["facets"].keys()) + + # Save the results first to file if user request + if filename or save: + if not filename: + filename = '{}-trends.json.gz'.format(query.replace(' ', '-')) + elif not filename.endswith('.json.gz'): + filename += '.json.gz' + + # Create/ append to the file + with helpers.open_file(filename) as fout: + for index, match in enumerate(results['matches']): + # Append facet info to make up a line + if result_facets: + match["facets"] = {} + for facet in result_facets: + match["facets"][facet] = results['facets'][facet][index]['values'] + + line = json.dumps(match) + '\n' + fout.write(line.encode('utf-8')) + + click.echo(click.style(u'Saved results into file {}'.format(filename), 'green')) + + # We buffer the entire output so we can use click's pager functionality + output = u'' + + # Output examples: + # - Facet by os + # 2017-06 + # os + # Linux 3.x 146,502 + # Windows 7 or 8 2,189 + # + # - Without facets + # 2017-06 19,799,459 + # 2017-07 21,077,099 + if result_facets: + for index, match in enumerate(results['matches']): + output += click.style(match['month'] + u'\n', fg='green') + if match['count'] > 0: + for facet in result_facets: + output += click.style(u' {}\n'.format(facet), fg='cyan') + for bucket in results['facets'][facet][index]['values']: + output += u' {:60}{}\n'.format(click.style(bucket['value'], bold=True), click.style(u'{:20,d}'.format(bucket['count']), fg='green')) + else: + output += u'{}\n'.format(click.style('N/A', bold=True)) + else: + # Without facets, show query total results over time + for index, match in enumerate(results['matches']): + output += u'{:20}{}\n'.format(click.style(match['month'], bold=True), click.style(u'{:20,d}'.format(match['count']), fg='green')) + + click.echo_via_pager(output) + + @main.command() @click.argument('ip', metavar='') def honeyscore(ip): @@ -706,7 +919,7 @@ def honeyscore(ip): click.echo(click.style('Not a honeypot', fg='green')) click.echo('Score: {}'.format(score)) - except: + except Exception: raise click.ClickException('Unable to calculate honeyscore') @@ -725,5 +938,12 @@ def radar(): except Exception as e: raise click.ClickException(u'{}'.format(e)) + +@main.command() +def version(): + """Print version of this tool.""" + print(pkg_resources.get_distribution("shodan").version) + + if __name__ == '__main__': main() diff --git a/shodan/alert.py b/shodan/alert.py deleted file mode 100644 index 7a89e90..0000000 --- a/shodan/alert.py +++ /dev/null @@ -1,9 +0,0 @@ -class Alert: - def __init__(self): - self.id = None - self.name = None - self.api_key = None - self.filters = None - self.credits = None - self.created = None - self.expires = None diff --git a/shodan/cli/alert.py b/shodan/cli/alert.py index a9f6ec8..1df11ea 100644 --- a/shodan/cli/alert.py +++ b/shodan/cli/alert.py @@ -1,7 +1,79 @@ import click +import csv +import gzip +import json import shodan +from tldextract import extract +from ipaddress import ip_address +from collections import defaultdict +from operator import itemgetter +from shodan import APIError from shodan.cli.helpers import get_api_key +from shodan.helpers import open_file, write_banner +from time import sleep + + +MAX_QUERY_LENGTH = 1000 + + +def aggregate_facet(api, networks, facets): + """Merge the results from multiple facet API queries into a single result object. + This is necessary because a user might be monitoring a lot of IPs/ networks so it doesn't fit + into a single API call. + """ + def _merge_custom_facets(lfacets, results): + for key in results['facets']: + if key not in lfacets: + lfacets[key] = defaultdict(int) + + for item in results['facets'][key]: + lfacets[key][item['value']] += item['count'] + + # We're going to create a custom facets dict where + # the key is the value of a facet. Normally the facets + # object is a list where each item has a "value" and "count" property. + tmp_facets = {} + count = 0 + + query = 'net:' + + for net in networks: + query += '{},'.format(net) + + # Start running API queries if the query length is getting long + if len(query) > MAX_QUERY_LENGTH: + results = api.count(query[:-1], facets=facets) + + _merge_custom_facets(tmp_facets, results) + count += results['total'] + query = 'net:' + + # Run any remaining search query + if query[-1] != ':': + results = api.count(query[:-1], facets=facets) + + _merge_custom_facets(tmp_facets, results) + count += results['total'] + + # Convert the internal facets structure back to the one that + # the API returns. + new_facets = {} + for facet in tmp_facets: + sorted_items = sorted(tmp_facets[facet].items(), key=itemgetter(1), reverse=True) + new_facets[facet] = [{'value': key, 'count': value} for key, value in sorted_items] + + # Make sure the facet keys exist even if there weren't any results + for facet, _ in facets: + if facet not in new_facets: + new_facets[facet] = [] + + return { + 'matches': [], + 'facets': new_facets, + 'total': count, + } + @click.group() def alert(): @@ -25,23 +97,263 @@ def alert_clear(): raise click.ClickException(e.value) click.echo("Alerts deleted") + @alert.command(name='create') @click.argument('name', metavar='') -@click.argument('netblock', metavar='') -def alert_create(name, netblock): +@click.argument('netblocks', metavar='', nargs=-1) +def alert_create(name, netblocks): """Create a network alert to monitor an external network""" key = get_api_key() # Get the list api = shodan.Shodan(key) try: - alert = api.create_alert(name, netblock) + alert = api.create_alert(name, netblocks) except shodan.APIError as e: raise click.ClickException(e.value) click.secho('Successfully created network alert!', fg='green') click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') + +@alert.command(name='domain') +@click.argument('domain', metavar='', type=str) +@click.option('--triggers', help='List of triggers to enable', default='malware,industrial_control_system,internet_scanner,iot,open_database,new_service,ssl_expired,vulnerable') +def alert_domain(domain, triggers): + """Create a network alert based on a domain name""" + key = get_api_key() + + api = shodan.Shodan(key) + try: + # Grab a list of IPs for the domain + domain = domain.lower() + domain_parse = extract(domain) + click.secho('Looking up domain information...', dim=True) + info = api.dns.domain_info(domain, type='A') + + if domain_parse.subdomain: + domain_ips = set([record['value'] for record in info['data'] + if record['subdomain'] == domain_parse.subdomain and + not ip_address(record['value']).is_private]) + else: + domain_ips = set([record['value'] for record in info['data'] + if not ip_address(record['value']).is_private]) + + if not domain_ips: + raise click.ClickException('No external IPs were found to be associated with this domain. ' + 'No alert was created.') + + # Create the actual alert + click.secho('Creating alert...', dim=True) + alert = api.create_alert('__domain: {}'.format(domain), list(domain_ips)) + + # Enable the triggers so it starts getting managed by Shodan Monitor + click.secho('Enabling triggers...', dim=True) + api.enable_alert_trigger(alert['id'], triggers) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully created domain alert!', fg='green') + click.secho('Alert ID: {}'.format(alert['id']), fg='cyan') + + +@alert.command(name='download') +@click.argument('filename', metavar='', type=str) +@click.option('--alert-id', help='Specific alert ID to download the data of', default=None) +def alert_download(filename, alert_id): + """Download all information for monitored networks/ IPs.""" + key = get_api_key() + + api = shodan.Shodan(key) + ips = set() + networks = set() + + # Helper method to process batches of IPs + def batch(iterable, size=1): + iter_length = len(iterable) + for ndx in range(0, iter_length, size): + yield iterable[ndx:min(ndx + size, iter_length)] + + try: + # Get the list of alerts for the user + click.echo('Looking up alert information...') + if alert_id: + alerts = [api.alerts(aid=alert_id.strip())] + else: + alerts = api.alerts() + + click.echo('Compiling list of networks/ IPs to download...') + for alert in alerts: + for net in alert['filters']['ip']: + if '/' in net: + networks.add(net) + else: + ips.add(net) + + click.echo('Downloading...') + with open_file(filename) as fout: + # Check if the user is able to use batch IP lookups + batch_size = 1 + if len(ips) > 0: + api_info = api.info() + if api_info['plan'] in ['corp', 'stream-100']: + batch_size = 100 + + # Convert it to a list so we can index into it + ips = list(ips) + + # Grab all the IP information + for ip in batch(ips, size=batch_size): + try: + click.echo(ip) + results = api.host(ip) + if not isinstance(results, list): + results = [results] + + for host in results: + for banner in host['data']: + write_banner(fout, banner) + except APIError: + pass + sleep(1) # Slow down a bit to make sure we don't hit the rate limit + + # Grab all the network ranges + for net in networks: + try: + counter = 0 + click.echo(net) + for banner in api.search_cursor('net:{}'.format(net)): + write_banner(fout, banner) + + # Slow down a bit to make sure we don't hit the rate limit + if counter % 100 == 0: + sleep(1) + counter += 1 + except APIError: + pass + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully downloaded results into: {}'.format(filename), fg='green') + + +@alert.command(name='export') +@click.option('--filename', help='Name of the output file', default='shodan-alerts.json.gz', type=str) +def alert_export(filename): + """Export the configuration of monitored networks/ IPs to be used by ``shodan alert import``.""" + # Setup the API wrapper + key = get_api_key() + api = shodan.Shodan(key) + + try: + # Get the list of alerts for the user + click.echo('Looking up alert information...') + alerts = api.alerts() + + # Create the output file + click.echo('Writing alerts to file: {}'.format(filename)) + with gzip.open(filename, 'wt', encoding='utf-8') as fout: + json.dump(alerts, fout) + except Exception as e: + raise click.ClickException(e.value) + + click.secho('Successfully exported monitored networks', fg='green') + + +@alert.command(name='import') +@click.argument('filename', metavar='') +def alert_import(filename): + """Export the configuration of monitored networks/ IPs to be used by ``shodan alert import``.""" + # Setup the API wrapper + key = get_api_key() + api = shodan.Shodan(key) + + # A mapping of the old notifier IDs to the new ones + notifier_map = {} + + try: + # Loading the alerts + click.echo('Loading alerts from: {}'.format(filename)) + with gzip.open(filename, 'rt', encoding='utf-8') as fin: + alerts = json.load(fin) + + for item in alerts: + # Create the alert + click.echo('Creating: {}'.format(item['name'])) + alert = api.create_alert(item['name'], item['filters']['ip']) + + # Enable any triggers + if item.get('triggers', {}): + triggers = ','.join(item['triggers'].keys()) + + api.enable_alert_trigger(alert['id'], triggers) + + # Add any whitelisted services for this trigger + for trigger, info in item['triggers'].items(): + if info.get('ignore', []): + for whitelist in info['ignore']: + api.ignore_alert_trigger_notification(alert['id'], trigger, whitelist['ip'], whitelist['port']) + + # Enable the notifiers + for prev_notifier in item.get('notifiers', []): + # We don't need to do anything for the default notifier as that + # uses the account's email address automatically. + if prev_notifier['id'] == 'default': + continue + + # Get the new notifier based on the ID of the old one + notifier = notifier_map.get(prev_notifier['id']) + + # Create the notifier if it doesn't yet exist + if notifier is None: + notifier = api.notifier.create(prev_notifier['provider'], prev_notifier['args'], description=prev_notifier['description']) + + # Add it to our map of old notifier IDs to new notifiers + notifier_map[prev_notifier['id']] = notifier + + api.add_alert_notifier(alert['id'], notifier['id']) + except Exception as e: + raise click.ClickException(e.value) + + click.secho('Successfully imported monitored networks', fg='green') + + +@alert.command(name='info') +@click.argument('alert', metavar='') +def alert_info(alert): + """Show information about a specific alert""" + key = get_api_key() + api = shodan.Shodan(key) + + try: + info = api.alerts(aid=alert) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho(info['name'], fg='cyan') + click.secho('Created: ', nl=False, dim=True) + click.secho(info['created'], fg='magenta') + + click.secho('Notifications: ', nl=False, dim=True) + if 'triggers' in info and info['triggers']: + click.secho('enabled', fg='green') + else: + click.echo('disabled') + + click.echo('') + click.secho('Network Range(s):', dim=True) + + for network in info['filters']['ip']: + click.echo(u' > {}'.format(click.style(network, fg='yellow'))) + + click.echo('') + if 'triggers' in info and info['triggers']: + click.secho('Triggers:', dim=True) + for trigger in info['triggers']: + click.echo(u' > {}'.format(click.style(trigger, fg='yellow'))) + click.echo('') + + @alert.command(name='list') @click.option('--expired', help='Whether or not to show expired alerts.', default=True, type=bool) def alert_list(expired): @@ -57,17 +369,21 @@ def alert_list(expired): if len(results) > 0: click.echo(u'# {:14} {:<21} {:<15s}'.format('Alert ID', 'Name', 'IP/ Network')) - # click.echo('#' * 65) + for alert in results: click.echo( u'{:16} {:<30} {:<35} '.format( - click.style(alert['id'], fg='yellow'), + click.style(alert['id'], fg='yellow'), click.style(alert['name'], fg='cyan'), click.style(', '.join(alert['filters']['ip']), fg='white') ), nl=False ) + if 'triggers' in alert and alert['triggers']: + click.secho('Triggers: ', fg='magenta', nl=False) + click.echo(', '.join(alert['triggers'].keys()), nl=False) + if 'expired' in alert and alert['expired']: click.secho('expired', fg='red') else: @@ -76,6 +392,95 @@ def alert_list(expired): click.echo("You haven't created any alerts yet.") +@alert.command(name='stats') +@click.option('--limit', help='The number of results to return.', default=10, type=int) +@click.option('--filename', '-O', help='Save the results in a CSV file of the provided name.', default=None) +@click.argument('facets', metavar='', nargs=-1) +def alert_stats(limit, filename, facets): + """Show summary information about your monitored networks""" + # Setup Shodan + key = get_api_key() + api = shodan.Shodan(key) + + # Make sure the user didn't supply an empty string + if not facets: + raise click.ClickException('No facets provided') + + facets = [(facet, limit) for facet in facets] + + # Get the list of IPs/ networks that the user is monitoring + networks = set() + try: + alerts = api.alerts() + for alert in alerts: + for tmp in alert['filters']['ip']: + networks.add(tmp) + except shodan.APIError as e: + raise click.ClickException(e.value) + + # Grab the facets the user requested + try: + results = aggregate_facet(api, networks, facets) + except shodan.APIError as e: + raise click.ClickException(e.value) + + # TODO: The below code was taken from __main__.py:stats() - we should refactor it so the code can be shared + # Print the stats tables + for facet in results['facets']: + click.echo('Top {} Results for Facet: {}'.format(len(results['facets'][facet]), facet)) + + for item in results['facets'][facet]: + # Force the value to be a string - necessary because some facet values are numbers + value = u'{}'.format(item['value']) + + click.echo(click.style(u'{:28s}'.format(value), fg='cyan'), nl=False) + click.echo(click.style(u'{:12,d}'.format(item['count']), fg='green')) + + click.echo('') + + # Create the output file if requested + fout = None + if filename: + if not filename.endswith('.csv'): + filename += '.csv' + fout = open(filename, 'w') + writer = csv.writer(fout, dialect=csv.excel) + + # Write the header that contains the facets + row = [] + for facet in results['facets']: + row.append(facet) + row.append('') + writer.writerow(row) + + # Every facet has 2 columns (key, value) + counter = 0 + has_items = True + while has_items: + # pylint: disable=W0612 + row = ['' for i in range(len(results['facets']) * 2)] + + pos = 0 + has_items = False + for facet in results['facets']: + values = results['facets'][facet] + + # Add the values for the facet into the current row + if len(values) > counter: + has_items = True + row[pos] = values[counter]['value'] + row[pos + 1] = values[counter]['count'] + + pos += 2 + + # Write out the row + if has_items: + writer.writerow(row) + + # Move to the next row of values + counter += 1 + + @alert.command(name='remove') @click.argument('alert_id', metavar='') def alert_remove(alert_id): @@ -89,3 +494,68 @@ def alert_remove(alert_id): except shodan.APIError as e: raise click.ClickException(e.value) click.echo("Alert deleted") + + +@alert.command(name='triggers') +def alert_list_triggers(): + """List the available notification triggers""" + key = get_api_key() + + # Get the list + api = shodan.Shodan(key) + try: + results = api.alert_triggers() + except shodan.APIError as e: + raise click.ClickException(e.value) + + if len(results) > 0: + click.secho('The following triggers can be enabled on alerts:', dim=True) + click.echo('') + + for trigger in sorted(results, key=itemgetter('name')): + click.secho('{:<12} '.format('Name'), dim=True, nl=False) + click.secho(trigger['name'], fg='yellow') + + click.secho('{:<12} '.format('Description'), dim=True, nl=False) + click.secho(trigger['description'], fg='cyan') + + click.secho('{:<12} '.format('Rule'), dim=True, nl=False) + click.echo(trigger['rule']) + + click.echo('') + else: + click.echo("No triggers currently available.") + + +@alert.command(name='enable') +@click.argument('alert_id', metavar='') +@click.argument('trigger', metavar='') +def alert_enable_trigger(alert_id, trigger): + """Enable a trigger for the alert""" + key = get_api_key() + + # Get the list + api = shodan.Shodan(key) + try: + api.enable_alert_trigger(alert_id, trigger) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully enabled the trigger: {}'.format(trigger), fg='green') + + +@alert.command(name='disable') +@click.argument('alert_id', metavar='') +@click.argument('trigger', metavar='') +def alert_disable_trigger(alert_id, trigger): + """Disable a trigger for the alert""" + key = get_api_key() + + # Get the list + api = shodan.Shodan(key) + try: + api.disable_alert_trigger(alert_id, trigger) + except shodan.APIError as e: + raise click.ClickException(e.value) + + click.secho('Successfully disabled the trigger: {}'.format(trigger), fg='green') diff --git a/shodan/cli/converter/__init__.py b/shodan/cli/converter/__init__.py index 507ca0b..08b068a 100644 --- a/shodan/cli/converter/__init__.py +++ b/shodan/cli/converter/__init__.py @@ -2,4 +2,4 @@ from .excel import ExcelConverter from .geojson import GeoJsonConverter from .images import ImagesConverter -from .kml import KmlConverter \ No newline at end of file +from .kml import KmlConverter diff --git a/shodan/cli/converter/base.py b/shodan/cli/converter/base.py index 9dc83c2..14b5f29 100644 --- a/shodan/cli/converter/base.py +++ b/shodan/cli/converter/base.py @@ -3,6 +3,6 @@ class Converter: def __init__(self, fout): self.fout = fout - + def process(self, fout): pass diff --git a/shodan/cli/converter/csvc.py b/shodan/cli/converter/csvc.py index c975695..2e4e2f2 100644 --- a/shodan/cli/converter/csvc.py +++ b/shodan/cli/converter/csvc.py @@ -2,7 +2,13 @@ from .base import Converter from ...helpers import iterate_files -from collections import MutableMapping +try: + # python 3.x: Import ABC from collections.abc + from collections.abc import MutableMapping +except ImportError: + # Python 2.x: Import ABC from collections + from collections import MutableMapping + from csv import writer as csv_writer, excel @@ -24,10 +30,13 @@ class CsvConverter(Converter): 'os', 'asn', 'port', + 'tags', + 'timestamp', 'transport', 'product', 'version', - + 'vulns', + 'ssl.cipher.version', 'ssl.cipher.bits', 'ssl.cipher.name', @@ -36,18 +45,23 @@ class CsvConverter(Converter): 'ssl.cert.serial', 'ssl.cert.fingerprint.sha1', 'ssl.cert.fingerprint.sha256', - + 'html', 'title', ] - - def process(self, files): - writer = csv_writer(self.fout, dialect=excel) - + + def process(self, files, file_size): + writer = csv_writer(self.fout, dialect=excel, lineterminator='\n') + # Write the header writer.writerow(self.fields) - + for banner in iterate_files(files): + # The "vulns" property can't be nicely flattened as-is so we turn + # it into a list before processing the banner. + if 'vulns' in banner: + banner['vulns'] = list(banner['vulns'].keys()) # Python3 returns dict_keys so we neeed to cover that to a list + try: row = [] for field in self.fields: @@ -56,33 +70,32 @@ def process(self, files): writer.writerow(row) except Exception: pass - + def banner_field(self, banner, flat_field): # The provided field is a collapsed form of the actual field fields = flat_field.split('.') - + try: current_obj = banner for field in fields: current_obj = current_obj[field] - + # Convert a list into a concatenated string if isinstance(current_obj, list): current_obj = ','.join([str(i) for i in current_obj]) - + return current_obj except Exception: pass - + return '' - + def flatten(self, d, parent_key='', sep='.'): items = [] for k, v in d.items(): new_key = parent_key + sep + k if parent_key else k if isinstance(v, MutableMapping): - # pylint: disable=E0602 - items.extend(flatten(v, new_key, sep=sep).items()) + items.extend(self.flatten(v, new_key, sep=sep).items()) else: items.append((new_key, v)) return dict(items) diff --git a/shodan/cli/converter/excel.py b/shodan/cli/converter/excel.py index a6b476d..2021a33 100644 --- a/shodan/cli/converter/excel.py +++ b/shodan/cli/converter/excel.py @@ -23,7 +23,7 @@ class ExcelConverter(Converter): 'transport', 'product', 'version', - + 'http.server', 'http.title', ] @@ -40,8 +40,8 @@ class ExcelConverter(Converter): 'http.server': 'Web Server', 'http.title': 'Website Title', } - - def process(self, files): + + def process(self, files, file_size): # Get the filename from the already-open file handle filename = self.fout.name @@ -51,18 +51,22 @@ def process(self, files): # Create the new workbook workbook = Workbook(filename) + # Check if Excel file is larger than 4GB + if file_size > 4e9: + workbook.use_zip64() + # Define some common styles/ formats bold = workbook.add_format({ 'bold': 1, }) - + # Create the main worksheet where all the raw data is shown main_sheet = workbook.add_worksheet('Raw Data') # Write the header - main_sheet.write(0, 0, 'IP', bold) # The IP field can be either ip_str or ipv6 so we treat it differently + main_sheet.write(0, 0, 'IP', bold) # The IP field can be either ip_str or ipv6 so we treat it differently main_sheet.set_column(0, 0, 20) - + row = 0 col = 1 for field in self.fields: @@ -80,7 +84,7 @@ def process(self, files): for field in self.fields: value = self.banner_field(banner, field) data.append(value) - + # Write those values to the main workbook # Starting off w/ the special "IP" property main_sheet.write_string(row, 0, get_ip(banner)) @@ -92,11 +96,11 @@ def process(self, files): row += 1 except Exception: pass - + # Aggregate summary information total += 1 ports[banner['port']] += 1 - + summary_sheet = workbook.add_worksheet('Summary') summary_sheet.write(0, 0, 'Total', bold) summary_sheet.write(0, 1, total) @@ -109,22 +113,24 @@ def process(self, files): summary_sheet.write(row, col, key) summary_sheet.write(row, col + 1, value) row += 1 - + + workbook.close() + def banner_field(self, banner, flat_field): # The provided field is a collapsed form of the actual field fields = flat_field.split('.') - + try: current_obj = banner for field in fields: current_obj = current_obj[field] - + # Convert a list into a concatenated string if isinstance(current_obj, list): current_obj = ','.join([str(i) for i in current_obj]) - + return current_obj except Exception: pass - + return '' diff --git a/shodan/cli/converter/geojson.py b/shodan/cli/converter/geojson.py index 8bde86f..83fb935 100644 --- a/shodan/cli/converter/geojson.py +++ b/shodan/cli/converter/geojson.py @@ -1,57 +1,52 @@ - +from json import dumps from .base import Converter from ...helpers import get_ip, iterate_files + class GeoJsonConverter(Converter): - + def header(self): self.fout.write("""{ "type": "FeatureCollection", "features": [ """) - + def footer(self): self.fout.write("""{ }]}""") - - def process(self, files): + + def process(self, files, file_size): # Write the header self.header() - - hosts = {} + + # We only want to generate 1 datapoint for each IP - not per service + unique_hosts = set() for banner in iterate_files(files): ip = get_ip(banner) if not ip: continue - - if ip not in hosts: - hosts[ip] = banner - hosts[ip]['ports'] = [] - - hosts[ip]['ports'].append(banner['port']) - - for ip, host in iter(hosts.items()): - self.write(host) - + + if ip not in unique_hosts: + self.write(ip, banner) + unique_hosts.add(ip) + self.footer() - - - def write(self, host): + + def write(self, ip, host): try: - ip = get_ip(host) lat, lon = host['location']['latitude'], host['location']['longitude'] - - feature = """{ - "type": "Feature", - "id": "{}", - "properties": { - "name": "{}" - }, - "geometry": { - "type": "Point", - "coordinates": [{}, {}] - } - }""".format(ip, ip, lat, lon) - - self.fout.write(feature) + feature = { + 'type': 'Feature', + 'id': ip, + 'properties': { + 'name': ip, + 'lat': lat, + 'lon': lon, + }, + 'geometry': { + 'type': 'Point', + 'coordinates': [lon, lat], + }, + } + self.fout.write(dumps(feature) + ',') except Exception: pass diff --git a/shodan/cli/converter/images.py b/shodan/cli/converter/images.py index b239b4d..fba9d11 100644 --- a/shodan/cli/converter/images.py +++ b/shodan/cli/converter/images.py @@ -14,8 +14,8 @@ class ImagesConverter(Converter): # special code in the Shodan CLI that relies on the "dirname" property to let # the user know where the images have been stored. dirname = None - - def process(self, files): + + def process(self, files, file_size): # Get the filename from the already-open file handle and use it as # the directory name to store the images. self.dirname = self.fout.name[:-7] + '-images' diff --git a/shodan/cli/converter/kml.py b/shodan/cli/converter/kml.py index 49938c2..9259ddf 100644 --- a/shodan/cli/converter/kml.py +++ b/shodan/cli/converter/kml.py @@ -2,38 +2,38 @@ from .base import Converter from ...helpers import iterate_files + class KmlConverter(Converter): - + def header(self): self.fout.write(""" """) - + def footer(self): self.fout.write("""""") - - def process(self, files): + + def process(self, files, file_size): # Write the header self.header() - + hosts = {} for banner in iterate_files(files): ip = banner.get('ip_str', banner.get('ipv6', None)) if not ip: continue - + if ip not in hosts: hosts[ip] = banner hosts[ip]['ports'] = [] - + hosts[ip]['ports'].append(banner['port']) - + for ip, host in iter(hosts.items()): self.write(host) - + self.footer() - - + def write(self, host): try: ip = host.get('ip_str', host.get('ipv6', None)) diff --git a/shodan/cli/data.py b/shodan/cli/data.py index 7cd7228..98d7852 100644 --- a/shodan/cli/data.py +++ b/shodan/cli/data.py @@ -27,6 +27,11 @@ def data_list(dataset): for file in files: click.echo(click.style(u'{:20s}'.format(file['name']), fg='cyan'), nl=False) click.echo(click.style('{:10s}'.format(helpers.humanize_bytes(file['size'])), fg='yellow'), nl=False) + + # Show the SHA1 checksum if available + if file.get('sha1'): + click.echo(click.style('{:42s}'.format(file['sha1']), fg='green'), nl=False) + click.echo('{}'.format(file['url'])) else: # If no dataset was provided then show a list of all datasets diff --git a/shodan/cli/helpers.py b/shodan/cli/helpers.py index 6ef9e1b..bde2f07 100644 --- a/shodan/cli/helpers.py +++ b/shodan/cli/helpers.py @@ -7,9 +7,16 @@ import itertools import os import sys +from ipaddress import ip_network, ip_address from .settings import SHODAN_CONFIG_DIR +try: + basestring # Python 2 +except NameError: + basestring = (str, ) # Python 3 + + def get_api_key(): '''Returns the API key of the current logged-in user.''' shodan_dir = os.path.expanduser(SHODAN_CONFIG_DIR) @@ -21,7 +28,8 @@ def get_api_key(): raise click.ClickException('Please run "shodan init " before using this command') # Make sure it is a read-only file - os.chmod(keyfile, 0o600) + if not oct(os.stat(keyfile).st_mode).endswith("600"): + os.chmod(keyfile, 0o600) with open(keyfile, 'r') as fin: return fin.read().strip() @@ -39,7 +47,7 @@ def timestr(): def open_streaming_file(directory, timestr, compresslevel=9): - return gzip.open('%s/%s.json.gz' % (directory, timestr), 'a', compresslevel) + return gzip.open('{}/{}.json.gz'.format(directory, timestr), 'a', compresslevel) def get_banner_field(banner, flat_field): @@ -57,9 +65,25 @@ def get_banner_field(banner, flat_field): return None +def filter_with_netmask(banner, netmask): + # filtering based on netmask is a more abstract concept than + # a mere check for a specific field and thus needs its own mechanism + # this will enable users to use the net:10.0.0.0/8 syntax they are used to + # to find specific networks from a big shodan download. + network = ip_network(netmask) + ip_field = get_banner_field(banner, 'ip') + if not ip_field: + return False + banner_ip_address = ip_address(ip_field) + return banner_ip_address in network + + def match_filters(banner, filters): for args in filters: flat_field, check = args.split(':', 1) + if flat_field == 'net': + return filter_with_netmask(banner, check) + value = get_banner_field(banner, flat_field) # If the field doesn't exist on the banner then ignore the record diff --git a/shodan/cli/host.py b/shodan/cli/host.py index befdc62..8ffdeed 100644 --- a/shodan/cli/host.py +++ b/shodan/cli/host.py @@ -64,9 +64,9 @@ def host_print_pretty(host, history=False): for port in ports: banner = { 'port': port, - 'transport': 'tcp', # All the filtered services use TCP - 'timestamp': host['data'][-1]['timestamp'], # Use the timestamp of the oldest banner - 'placeholder': True, # Don't store this banner when the file is saved + 'transport': 'tcp', # All the filtered services use TCP + 'timestamp': host['data'][-1]['timestamp'], # Use the timestamp of the oldest banner + 'placeholder': True, # Don't store this banner when the file is saved } host['data'].append(banner) @@ -91,10 +91,22 @@ def host_print_pretty(host, history=False): click.echo(click.style('\t\t({})'.format(date), fg='white', dim=True), nl=False) click.echo('') + # Show optional HTTP information + if 'http' in banner: + if 'title' in banner['http'] and banner['http']['title']: + click.echo('\t|-- HTTP title: {}'.format(banner['http']['title'])) + # Show optional ssl info if 'ssl' in banner: + if 'cert' in banner['ssl'] and banner['ssl']['cert']: + if 'issuer' in banner['ssl']['cert'] and banner['ssl']['cert']['issuer']: + issuer = ', '.join(['{}={}'.format(key, value) for key, value in banner['ssl']['cert']['issuer'].items()]) + click.echo('\t|-- Cert Issuer: {}'.format(issuer)) + if 'subject' in banner['ssl']['cert'] and banner['ssl']['cert']['subject']: + subject = ', '.join(['{}={}'.format(key, value) for key, value in banner['ssl']['cert']['subject'].items()]) + click.echo('\t|-- Cert Subject: {}'.format(subject)) if 'versions' in banner['ssl'] and banner['ssl']['versions']: - click.echo('\t|-- SSL Versions: {}'.format(', '.join([version for version in sorted(banner['ssl']['versions']) if not version.startswith('-')]))) + click.echo('\t|-- SSL Versions: {}'.format(', '.join([item for item in sorted(banner['ssl']['versions']) if not version.startswith('-')]))) if 'dhparams' in banner['ssl'] and banner['ssl']['dhparams']: click.echo('\t|-- Diffie-Hellman Parameters:') click.echo('\t\t{:15s}{}\n\t\t{:15s}{}'.format('Bits:', banner['ssl']['dhparams']['bits'], 'Generator:', banner['ssl']['dhparams']['generator'])) @@ -119,4 +131,4 @@ def host_print_tsv(host, history=False): HOST_PRINT = { 'pretty': host_print_pretty, 'tsv': host_print_tsv, -} \ No newline at end of file +} diff --git a/shodan/cli/organization.py b/shodan/cli/organization.py index 50d814c..5fbb764 100644 --- a/shodan/cli/organization.py +++ b/shodan/cli/organization.py @@ -22,7 +22,7 @@ def add(silent, user): api.org.add_member(user, notify=not silent) except shodan.APIError as e: raise click.ClickException(e.value) - + click.secho('Successfully added the new member', fg='green') @@ -39,11 +39,11 @@ def info(): click.secho(organization['name'], fg='cyan') click.secho('Access Level: ', nl=False, dim=True) click.secho(humanize_api_plan(organization['upgrade_type']), fg='magenta') - + if organization['domains']: click.secho('Authorized Domains: ', nl=False, dim=True) click.echo(', '.join(organization['domains'])) - + click.echo('') click.secho('Administrators:', dim=True) @@ -51,8 +51,8 @@ def info(): click.echo(u' > {:30}\t{:30}'.format( click.style(admin['username'], fg='yellow'), admin['email']) - ) - + ) + click.echo('') if organization['members']: click.secho('Members:', dim=True) @@ -76,5 +76,5 @@ def remove(user): api.org.remove_member(user) except shodan.APIError as e: raise click.ClickException(e.value) - + click.secho('Successfully removed the member', fg='green') diff --git a/shodan/cli/scan.py b/shodan/cli/scan.py index 5220339..cfc7aab 100644 --- a/shodan/cli/scan.py +++ b/shodan/cli/scan.py @@ -17,6 +17,35 @@ def scan(): pass +@scan.command(name='list') +def scan_list(): + """Show recently launched scans""" + key = get_api_key() + + # Get the list + api = shodan.Shodan(key) + try: + scans = api.scans() + except shodan.APIError as e: + raise click.ClickException(e.value) + + if len(scans) > 0: + click.echo(u'# {} Scans Total - Showing 10 most recent scans:'.format(scans['total'])) + click.echo(u'# {:20} {:<15} {:<10} {:<15s}'.format('Scan ID', 'Status', 'Size', 'Timestamp')) + # click.echo('#' * 65) + for scan in scans['matches'][:10]: + click.echo( + u'{:31} {:<24} {:<10} {:<15s}'.format( + click.style(scan['id'], fg='yellow'), + click.style(scan['status'], fg='cyan'), + scan['size'], + scan['created'] + ) + ) + else: + click.echo("You haven't yet launched any scans.") + + @scan.command(name='internet') @click.option('--quiet', help='Disable the printing of information to the screen.', default=False, is_flag=True) @click.argument('port', type=int) @@ -58,12 +87,11 @@ def scan_internet(quiet, port, protocol): if not quiet: click.echo('{0:<40} {1:<20} {2}'.format( - click.style(helpers.get_ip(banner), fg=COLORIZE_FIELDS['ip_str']), - click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']), - ';'.join(banner['hostnames']) - ) + click.style(helpers.get_ip(banner), fg=COLORIZE_FIELDS['ip_str']), + click.style(str(banner['port']), fg=COLORIZE_FIELDS['port']), + ';'.join(banner['hostnames'])) ) - except shodan.APIError as e: + except shodan.APIError: # We stop waiting for results if the scan has been processed by the crawlers and # there haven't been new results in a while if done: @@ -72,7 +100,7 @@ def scan_internet(quiet, port, protocol): scan = api.scan_status(scan['id']) if scan['status'] == 'DONE': done = True - except socket.timeout as e: + except socket.timeout: # We stop waiting for results if the scan has been processed by the crawlers and # there haven't been new results in a while if done: @@ -129,6 +157,7 @@ def scan_submit(wait, filename, force, verbose, netblocks): # Return immediately if wait <= 0: + click.echo('Scan ID: {}'.format(scan['id'])) click.echo('Exiting now, not waiting for results. Use the API or website to retrieve the results of the scan.') else: # Setup an alert to wait for responses @@ -177,7 +206,7 @@ def scan_submit(wait, filename, force, verbose, netblocks): done = True break - except shodan.APIError as e: + except shodan.APIError: # If the connection timed out before the timeout, that means the streaming server # that the user tried to reach is down. In that case, lets wait briefly and try # to connect again! @@ -195,7 +224,7 @@ def scan_submit(wait, filename, force, verbose, netblocks): if verbose: click.echo('# Scan status: {}'.format(scan['status'])) - except socket.timeout as e: + except socket.timeout: # If the connection timed out before the timeout, that means the streaming server # that the user tried to reach is down. In that case, lets wait a second and try # to connect again! diff --git a/shodan/cli/settings.py b/shodan/cli/settings.py index 6f27e5d..05c1b9f 100644 --- a/shodan/cli/settings.py +++ b/shodan/cli/settings.py @@ -1,5 +1,11 @@ -SHODAN_CONFIG_DIR = '~/.shodan/' +from os import path + +if path.exists(path.expanduser("~/.shodan")): + SHODAN_CONFIG_DIR = '~/.shodan/' +else: + SHODAN_CONFIG_DIR = "~/.config/shodan/" + COLORIZE_FIELDS = { 'ip_str': 'green', 'port': 'yellow', diff --git a/shodan/cli/worldmap.py b/shodan/cli/worldmap.py index 60ef075..4e09872 100755 --- a/shodan/cli/worldmap.py +++ b/shodan/cli/worldmap.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python ''' F-Secure Virus World Map console edition @@ -30,8 +29,7 @@ 'coords': [90.0, -180.0, -90.0, 180.0], # PyLint freaks out about the world map backslashes so ignore those warnings - # pylint: disable=W1401 - 'data': ''' + 'data': r''' . _..::__: ,-"-"._ |7 , _,.__ _.___ _ _<_>`!(._`.`-. / _._ `_ ,_/ ' '-._.---.-.__ .{ " " `-==,',._\{ \ / {) / _ ">_,-' ` mt-2_ @@ -108,14 +106,14 @@ def latlon_to_coords(self, lat, lon): TODO: filter out stuff that doesn't fit TODO: make it possible to use "zoomed" maps """ - width = (self.corners[3]-self.corners[1]) - height = (self.corners[2]-self.corners[0]) + width = (self.corners[3] - self.corners[1]) + height = (self.corners[2] - self.corners[0]) # change to 0-180, 0-360 - abs_lat = -lat+90 - abs_lon = lon+180 - x = (abs_lon/360.0)*width + self.corners[1] - y = (abs_lat/180.0)*height + self.corners[0] + abs_lat = -lat + 90 + abs_lon = lon + 180 + x = (abs_lon / 360.0) * width + self.corners[1] + y = (abs_lat / 180.0) * height + self.corners[0] return int(x), int(y) def set_data(self, data): @@ -155,12 +153,12 @@ def draw(self, target): self.window.addstr(0, 0, self.map) # FIXME: position to be defined in map config? - row = self.corners[2]-6 + row = self.corners[2] - 6 items_to_show = 5 for lat, lon, char, desc, attrs, color in self.data: # to make this work almost everywhere. see http://docs.python.org/2/library/curses.html if desc: - desc = desc.encode(self.encoding, 'ignore') + desc = desc.encode(self.encoding, 'ignore').decode() if items_to_show <= 0: break char_x, char_y = self.latlon_to_coords(lat, lon) @@ -168,7 +166,7 @@ def draw(self, target): attrs |= curses.color_pair(self.colors[color]) self.window.addstr(char_y, char_x, char, attrs) if desc: - det_show = "%s %s" % (char, desc) + det_show = "{} {}".format(char, desc) else: det_show = None @@ -177,11 +175,11 @@ def draw(self, target): self.window.addstr(row, 1, det_show, attrs) row += 1 items_to_show -= 1 - except StandardError: + except Exception: # FIXME: check window size before addstr() break self.window.overwrite(target) - self.window.leaveok(1) + self.window.leaveok(True) class MapApp(object): @@ -257,6 +255,7 @@ def main(argv=None): api = Shodan(get_api_key()) return launch_map(api) + if __name__ == '__main__': import sys sys.exit(main()) diff --git a/shodan/client.py b/shodan/client.py index 0fd9167..21c70af 100644 --- a/shodan/client.py +++ b/shodan/client.py @@ -7,13 +7,15 @@ :copyright: (c) 2014- by John Matherly """ +import math +import os import time import requests import json from .exception import APIError -from .helpers import api_request, create_facet_string +from .helpers import create_facet_string from .stream import Stream @@ -64,6 +66,89 @@ def list_files(self, dataset): """ return self.parent._request('/shodan/data/{}'.format(dataset), {}) + class Dns: + + def __init__(self, parent): + self.parent = parent + + def domain_info(self, domain, history=False, type=None, page=1): + """Grab the DNS information for a domain. + """ + args = { + 'page': page, + } + if history: + args['history'] = history + if type: + args['type'] = type + return self.parent._request('/dns/domain/{}'.format(domain), args) + + class Notifier: + + def __init__(self, parent): + self.parent = parent + + def create(self, provider, args, description=None): + """Get the settings for the specified notifier that a user has configured. + + :param provider: Provider name + :type provider: str + :param args: Provider arguments + :type args: dict + :param description: Human-friendly description of the notifier + :type description: str + :returns: dict -- fields are 'success' and 'id' of the notifier + """ + args['provider'] = provider + + if description: + args['description'] = description + + return self.parent._request('/notifier', args, method='post') + + def edit(self, nid, args): + """Get the settings for the specified notifier that a user has configured. + + :param nid: Notifier ID + :type nid: str + :param args: Provider arguments + :type args: dict + :returns: dict -- fields are 'success' and 'id' of the notifier + """ + return self.parent._request('/notifier/{}'.format(nid), args, method='put') + + def get(self, nid): + """Get the settings for the specified notifier that a user has configured. + + :param nid: Notifier ID + :type nid: str + :returns: dict -- object describing the notifier settings + """ + return self.parent._request('/notifier/{}'.format(nid), {}) + + def list_notifiers(self): + """Returns a list of notifiers that the user has added. + + :returns: A list of notifierse that are available on the account + """ + return self.parent._request('/notifier', {}) + + def list_providers(self): + """Returns a list of supported notification providers. + + :returns: A list of providers where each object describes a provider + """ + return self.parent._request('/notifier/provider', {}) + + def remove(self, nid): + """Delete the provided notifier. + + :param nid: Notifier ID + :type nid: str + :returns: dict -- 'success' set to True if action succeeded + """ + return self.parent._request('/notifier/{}'.format(nid), {}, method='delete') + class Tools: def __init__(self, parent): @@ -170,28 +255,75 @@ def remove_member(self, user): """ return self.parent._request('/org/member/{}'.format(user), {}, method='DELETE')['success'] + class Trends: + + def __init__(self, parent): + self.parent = parent + + def search(self, query, facets): + """Search the Shodan historical database. + + :param query: Search query; identical syntax to the website + :type query: str + :param facets: (optional) A list of properties to get summary information on + :type facets: str + + :returns: A dictionary with 3 main items: matches, facets and total. Visit the website for more detailed information. + """ + args = { + 'query': query, + 'facets': create_facet_string(facets), + } + + return self.parent._request('/api/v1/search', args, service='trends') + + def search_facets(self): + """This method returns a list of facets that can be used to get a breakdown of the top values for a property. + + :returns: A list of strings where each is a facet name + """ + return self.parent._request('/api/v1/search/facets', {}, service='trends') + + def search_filters(self): + """This method returns a list of search filters that can be used in the search query. + + :returns: A list of strings where each is a filter name + """ + return self.parent._request('/api/v1/search/filters', {}, service='trends') + def __init__(self, key, proxies=None): """Initializes the API object. :param key: The Shodan API key. :type key: str :param proxies: A proxies array for the requests library, e.g. {'https': 'your proxy'} - :type key: dict + :type proxies: dict """ self.api_key = key self.base_url = 'https://api.shodan.io' self.base_exploits_url = 'https://exploits.shodan.io' + self.base_trends_url = 'https://trends.shodan.io' self.data = self.Data(self) + self.dns = self.Dns(self) self.exploits = self.Exploits(self) + self.trends = self.Trends(self) self.labs = self.Labs(self) + self.notifier = self.Notifier(self) self.org = self.Organization(self) self.tools = self.Tools(self) self.stream = Stream(key, proxies=proxies) self._session = requests.Session() + self.api_rate_limit = 1 # Requests per second + self._api_query_time = None + if proxies: self._session.proxies.update(proxies) + self._session.trust_env = False + + if os.environ.get('SHODAN_API_URL'): + self.base_url = os.environ.get('SHODAN_API_URL') - def _request(self, function, params, service='shodan', method='get'): + def _request(self, function, params, service='shodan', method='get', json_data=None): """General-purpose function to create web requests to SHODAN. Arguments: @@ -209,19 +341,32 @@ def _request(self, function, params, service='shodan', method='get'): base_url = { 'shodan': self.base_url, 'exploits': self.base_exploits_url, + 'trends': self.base_trends_url, }.get(service, 'shodan') + # Wait for API rate limit + if self._api_query_time is not None and self.api_rate_limit > 0: + while (1.0 / self.api_rate_limit) + self._api_query_time >= time.time(): + time.sleep(0.1 / self.api_rate_limit) + # Send the request try: method = method.lower() if method == 'post': - data = self._session.post(base_url + function, params) + if json_data: + data = self._session.post(base_url + function, params=params, + data=json.dumps(json_data), + headers={'content-type': 'application/json'}, + ) + else: + data = self._session.post(base_url + function, params) elif method == 'put': data = self._session.put(base_url + function, params=params) elif method == 'delete': data = self._session.delete(base_url + function, params=params) else: data = self._session.get(base_url + function, params=params) + self._api_query_time = time.time() except Exception: raise APIError('Unable to connect to Shodan') @@ -242,6 +387,8 @@ def _request(self, function, params, service='shodan', method='get'): raise APIError(error) elif data.status_code == 403: raise APIError('Access denied (403 Forbidden)') + elif data.status_code == 502: + raise APIError('Bad Gateway (502)') # Parse the text into JSON try: @@ -291,7 +438,7 @@ def host(self, ips, history=False, minify=False): params['history'] = history if minify: params['minify'] = minify - return self._request('/shodan/host/%s' % ','.join(ips), params) + return self._request('/shodan/host/{}'.format(','.join(ips)), params) def info(self): """Returns information about the current API key, such as a list of add-ons @@ -347,6 +494,16 @@ def scan(self, ips, force=False): return self._request('/shodan/scan', params, method='post') + def scans(self, page=1): + """Get a list of scans submitted + + :param page: Page through the list of scans 100 results at a time + :type page: int + """ + return self._request('/shodan/scans', { + 'page': page, + }) + def scan_internet(self, port, protocol): """Scan a network using Shodan @@ -372,9 +529,9 @@ def scan_status(self, scan_id): :returns: A dictionary with general information about the scan, including its status in getting processed. """ - return self._request('/shodan/scan/%s' % scan_id, {}) + return self._request('/shodan/scan/{}'.format(scan_id), {}) - def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True): + def search(self, query, page=1, limit=None, offset=None, facets=None, minify=True, fields=None): """Search the SHODAN database. :param query: Search query; identical syntax to the website @@ -389,6 +546,8 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru :type facets: str :param minify: (optional) Whether to minify the banner and only return the important data :type minify: bool + :param fields: (optional) List of properties that should get returned. This option is mutually exclusive with the "minify" parameter + :type fields: str :returns: A dictionary with 2 main items: matches and total. If facets have been provided then another property called "facets" will be available at the top-level of the dictionary. Visit the website for more detailed information. """ @@ -406,9 +565,12 @@ def search(self, query, page=1, limit=None, offset=None, facets=None, minify=Tru if facets: args['facets'] = create_facet_string(facets) + if fields and isinstance(fields, list): + args['fields'] = ','.join(fields) + return self._request('/shodan/host/search', args) - def search_cursor(self, query, minify=True, retries=5): + def search_cursor(self, query, minify=True, retries=5, fields=None): """Search the SHODAN database. This method returns an iterator that can directly be in a loop. Use it when you want to loop over @@ -421,33 +583,58 @@ def search_cursor(self, query, minify=True, retries=5): :param minify: (optional) Whether to minify the banner and only return the important data :type minify: bool :param retries: (optional) How often to retry the search in case it times out - :type minify: int + :type retries: int :returns: A search cursor that can be used as an iterator/ generator. """ page = 1 + total_pages = 0 tries = 0 - results = { - 'matches': [], - 'total': None, - } - while page == 1 or results['matches']: + + # Grab the initial page and use the total to calculate the expected number of pages + results = self.search(query, minify=minify, page=page, fields=fields) + if results['total']: + total_pages = int(math.ceil(results['total'] / 100)) + + for banner in results['matches']: + try: + yield banner + except GeneratorExit: + return # exit out of the function + page += 1 + + # Keep iterating over the results from page 2 onwards + while page <= total_pages: try: - results = self.search(query, minify=minify, page=page) + results = self.search(query, minify=minify, page=page, fields=fields) for banner in results['matches']: try: yield banner except GeneratorExit: - return # exit out of the function + return # exit out of the function page += 1 tries = 0 except Exception: # We've retried several times but it keeps failing, so lets error out if tries >= retries: - break + raise APIError('Retry limit reached ({:d})'.format(retries)) tries += 1 - time.sleep(1.0) # wait 1 second if the search errored out for some reason + time.sleep(tries) # wait (1 second * retry number) if the search errored out for some reason + + def search_facets(self): + """Returns a list of search facets that can be used to get aggregate information about a search query. + + :returns: A list of strings where each is a facet name + """ + return self._request('/shodan/host/search/facets', {}) + + def search_filters(self): + """Returns a list of search filters that are available. + + :returns: A list of strings where each is a filter name + """ + return self._request('/shodan/host/search/filters', {}) def search_tokens(self, query): """Returns information about the search query itself (filters used etc.) @@ -507,8 +694,8 @@ def queries_search(self, query, page=1): def queries_tags(self, size=10): """Search the directory of saved search queries in Shodan. - :param query: The number of tags to return - :type page: int + :param size: The number of tags to return + :type size: int :returns: A list of tags. """ @@ -518,12 +705,14 @@ def queries_tags(self, size=10): return self._request('/shodan/query/tags', args) def create_alert(self, name, ip, expires=0): - """Search the directory of saved search queries in Shodan. + """Create a network alert/ private firehose for the specified IP range(s) - :param query: The number of tags to return - :type page: int + :param name: Name of the alert + :type name: str + :param ip: Network range(s) to monitor + :type ip: str OR list of str - :returns: A list of tags. + :returns: A dict describing the alert """ data = { 'name': name, @@ -533,31 +722,84 @@ def create_alert(self, name, ip, expires=0): 'expires': expires, } - response = api_request(self.api_key, '/shodan/alert', data=data, params={}, method='post', - proxies=self._session.proxies) + response = self._request('/shodan/alert', params={}, json_data=data, method='post') + + return response + + def edit_alert(self, aid, ip): + """Edit the IPs that should be monitored by the alert. + + :param aid: Alert ID + :type name: str + :param ip: Network range(s) to monitor + :type ip: str OR list of str + + :returns: A dict describing the alert + """ + data = { + 'filters': { + 'ip': ip, + }, + } + + response = self._request('/shodan/alert/{}'.format(aid), params={}, json_data=data, method='post') return response def alerts(self, aid=None, include_expired=True): """List all of the active alerts that the user created.""" if aid: - func = '/shodan/alert/%s/info' % aid + func = '/shodan/alert/{}/info'.format(aid) else: func = '/shodan/alert/info' - response = api_request(self.api_key, func, params={ + response = self._request(func, params={ 'include_expired': include_expired, - }, - proxies=self._session.proxies) + }) return response def delete_alert(self, aid): """Delete the alert with the given ID.""" - func = '/shodan/alert/%s' % aid + func = '/shodan/alert/{}'.format(aid) - response = api_request(self.api_key, func, params={}, method='delete', - proxies=self._session.proxies) + response = self._request(func, params={}, method='delete') return response + def alert_triggers(self): + """Return a list of available triggers that can be enabled for alerts. + + :returns: A list of triggers + """ + return self._request('/shodan/alert/triggers', {}) + + def enable_alert_trigger(self, aid, trigger): + """Enable the given trigger on the alert.""" + return self._request('/shodan/alert/{}/trigger/{}'.format(aid, trigger), {}, method='put') + + def disable_alert_trigger(self, aid, trigger): + """Disable the given trigger on the alert.""" + return self._request('/shodan/alert/{}/trigger/{}'.format(aid, trigger), {}, method='delete') + + def ignore_alert_trigger_notification(self, aid, trigger, ip, port, vulns=None): + """Ignore trigger notifications for the provided IP and port.""" + # The "vulnerable" and "vulnerable_unverified" triggers let you specify specific vulnerabilities + # to ignore. If a user provides a "vulns" list and specifies on of those triggers then we'll use + # a different API endpoint. + if trigger in ('vulnerable', 'vulnerable_unverified') and vulns and isinstance(vulns, list): + return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}/{}'.format(aid, trigger, ip, port, ','.join(vulns)), {}, method='put') + + return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='put') + + def unignore_alert_trigger_notification(self, aid, trigger, ip, port): + """Re-enable trigger notifications for the provided IP and port""" + return self._request('/shodan/alert/{}/trigger/{}/ignore/{}:{}'.format(aid, trigger, ip, port), {}, method='delete') + + def add_alert_notifier(self, aid, nid): + """Enable the given notifier for an alert that has triggers enabled.""" + return self._request('/shodan/alert/{}/notifier/{}'.format(aid, nid), {}, method='put') + + def remove_alert_notifier(self, aid, nid): + """Remove the given notifier for an alert that has triggers enabled.""" + return self._request('/shodan/alert/{}/notifier/{}'.format(aid, nid), {}, method='delete') diff --git a/shodan/exception.py b/shodan/exception.py index c4878b1..75b158e 100644 --- a/shodan/exception.py +++ b/shodan/exception.py @@ -2,11 +2,10 @@ class APIError(Exception): """This exception gets raised whenever a non-200 status code was returned by the Shodan API.""" def __init__(self, value): self.value = value - + def __str__(self): return self.value class APITimeout(APIError): - pass - + pass diff --git a/shodan/helpers.py b/shodan/helpers.py index d3c4a1f..563ecb4 100644 --- a/shodan/helpers.py +++ b/shodan/helpers.py @@ -19,7 +19,7 @@ def create_facet_string(facets): if isinstance(facet, basestring): facet_str += facet else: - facet_str += '%s:%s' % (facet[0], facet[1]) + facet_str += '{}:{}'.format(facet[0], facet[1]) facet_str += ',' return facet_str[:-1] @@ -76,7 +76,7 @@ def api_request(key, function, params=None, data=None, base_url='https://api.sho # Parse the text into JSON try: data = data.json() - except: + except Exception: raise APIError('Unable to parse JSON response') # Raise an exception if an error occurred @@ -113,15 +113,20 @@ def iterate_files(files, fast=False): for line in fin: # Ensure the line has been decoded into a string to prevent errors w/ Python3 - line = line.decode('utf-8') + if not isinstance(line, basestring): + line = line.decode('utf-8') # Convert the JSON into a native Python object banner = loads(line) yield banner + def get_screenshot(banner): - if 'opts' in banner and 'screenshot' in banner['opts']: + if 'screenshot' in banner and banner['screenshot']: + return banner['screenshot'] + elif 'opts' in banner and 'screenshot' in banner['opts']: return banner['opts']['screenshot'] + return None @@ -140,7 +145,7 @@ def write_banner(fout, banner): fout.write(line.encode('utf-8')) -def humanize_bytes(bytes, precision=1): +def humanize_bytes(byte_count, precision=1): """Return a humanized string representation of a number of bytes. >>> humanize_bytes(1) '1 byte' @@ -159,16 +164,15 @@ def humanize_bytes(bytes, precision=1): >>> humanize_bytes(1024*1234*1111,1) '1.3 GB' """ - - if bytes == 1: + if byte_count == 1: return '1 byte' - if bytes < 1024: - return '%.*f %s' % (precision, bytes, "bytes") + if byte_count < 1024: + return '{0:0.{1}f} {2}'.format(byte_count, 0, 'bytes') suffixes = ['KB', 'MB', 'GB', 'TB', 'PB'] - multiple = 1024.0 #.0 force float on python 2 + multiple = 1024.0 # .0 to force float on python 2 for suffix in suffixes: - bytes /= multiple - if bytes < multiple: - return '%.*f %s' % (precision, bytes, suffix) - return '%.*f %s' % (precision, bytes, suffix) + byte_count /= multiple + if byte_count < multiple: + return '{0:0.{1}f} {2}'.format(byte_count, precision, suffix) + return '{0:0.{1}f} {2}'.format(byte_count, precision, suffix) diff --git a/shodan/stream.py b/shodan/stream.py index 49ab633..9900b08 100644 --- a/shodan/stream.py +++ b/shodan/stream.py @@ -13,7 +13,7 @@ def __init__(self, api_key, proxies=None): self.api_key = api_key self.proxies = proxies - def _create_stream(self, name, timeout=None): + def _create_stream(self, name, query=None, timeout=None): params = { 'key': self.api_key, } @@ -21,7 +21,7 @@ def _create_stream(self, name, timeout=None): # The user doesn't want to use a timeout # If the timeout is specified as 0 then we also don't want to have a timeout - if ( timeout and timeout <= 0 ) or ( timeout == 0 ): + if (timeout and timeout <= 0) or (timeout == 0): timeout = None # If the user requested a timeout then we need to disable heartbeat messages @@ -30,6 +30,9 @@ def _create_stream(self, name, timeout=None): if timeout: params['heartbeat'] = False + if query is not None: + params['query'] = query + try: while True: req = requests.get(stream_url, params=params, stream=True, timeout=timeout, @@ -43,16 +46,16 @@ def _create_stream(self, name, timeout=None): # not specific to Cloudflare. if req.status_code != 524 or timeout >= 0: break - except Exception as e: + except Exception: raise APIError('Unable to contact the Shodan Streaming API') if req.status_code != 200: try: data = json.loads(req.text) raise APIError(data['error']) - except APIError as e: + except APIError: raise - except Exception as e: + except Exception: pass raise APIError('Invalid API key or you do not have access to the Streaming API') if req.encoding is None: @@ -60,7 +63,7 @@ def _create_stream(self, name, timeout=None): return req def _iter_stream(self, stream, raw): - for line in stream.iter_lines(decode_unicode=True): + for line in stream.iter_lines(): # The Streaming API sends out heartbeat messages that are newlines # We want to ignore those messages since they don't contain any data if line: @@ -71,16 +74,16 @@ def _iter_stream(self, stream, raw): def alert(self, aid=None, timeout=None, raw=False): if aid: - stream = self._create_stream('/shodan/alert/%s' % aid, timeout=timeout) + stream = self._create_stream('/shodan/alert/{}'.format(aid), timeout=timeout) else: stream = self._create_stream('/shodan/alert', timeout=timeout) try: for line in self._iter_stream(stream, raw): yield line - except requests.exceptions.ConnectionError as e: + except requests.exceptions.ConnectionError: raise APIError('Stream timed out') - except ssl.SSLError as e: + except ssl.SSLError: raise APIError('Stream timed out') def asn(self, asn, raw=False, timeout=None): @@ -90,7 +93,7 @@ def asn(self, asn, raw=False, timeout=None): :param asn: A list of ASN to return banner data on. :type asn: string[] """ - stream = self._create_stream('/shodan/asn/%s' % ','.join(asn), timeout=timeout) + stream = self._create_stream('/shodan/asn/{}'.format(','.join(asn)), timeout=timeout) for line in self._iter_stream(stream, raw): yield line @@ -109,7 +112,19 @@ def countries(self, countries, raw=False, timeout=None): :param countries: A list of countries to return banner data on. :type countries: string[] """ - stream = self._create_stream('/shodan/countries/%s' % ','.join(countries), timeout=timeout) + stream = self._create_stream('/shodan/countries/{}'.format(','.join(countries)), timeout=timeout) + for line in self._iter_stream(stream, raw): + yield line + + def custom(self, query, raw=False, timeout=None): + """ + A filtered version of the "banners" stream to only return banners that match the query of interest. The query + can vary and mix-match with different arguments (ports, tags, vulns, etc). + + :param query: A space-separated list of key:value filters query to return banner data on. + :type query: string + """ + stream = self._create_stream('/shodan/custom', query=query, timeout=timeout) for line in self._iter_stream(stream, raw): yield line @@ -120,7 +135,28 @@ def ports(self, ports, raw=False, timeout=None): :param ports: A list of ports to return banner data on. :type ports: int[] """ - stream = self._create_stream('/shodan/ports/%s' % ','.join([str(port) for port in ports]), timeout=timeout) + stream = self._create_stream('/shodan/ports/{}'.format(','.join([str(port) for port in ports])), timeout=timeout) + for line in self._iter_stream(stream, raw): + yield line + + def tags(self, tags, raw=False, timeout=None): + """ + A filtered version of the "banners" stream to only return banners that match the tags of interest. + + :param tags: A list of tags to return banner data on. + :type tags: string[] + """ + stream = self._create_stream('/shodan/tags/{}'.format(','.join(tags)), timeout=timeout) for line in self._iter_stream(stream, raw): yield line + def vulns(self, vulns, raw=False, timeout=None): + """ + A filtered version of the "banners" stream to only return banners that match the vulnerabilities of interest. + + :param vulns: A list of vulns to return banner data on. + :type vulns: string[] + """ + stream = self._create_stream('/shodan/vulns/{}'.format(','.join(vulns)), timeout=timeout) + for line in self._iter_stream(stream, raw): + yield line diff --git a/shodan/threatnet.py b/shodan/threatnet.py index 97c0c7e..cad9bdd 100644 --- a/shodan/threatnet.py +++ b/shodan/threatnet.py @@ -24,13 +24,13 @@ def _create_stream(self, name): try: req = requests.get(self.base_url + name, params={'key': self.parent.api_key}, stream=True, proxies=self.proxies) - except: + except Exception: raise APIError('Unable to contact the Shodan Streaming API') if req.status_code != 200: try: raise APIError(req.json()['error']) - except: + except Exception: pass raise APIError('Invalid API key or you do not have access to the Streaming API') return req @@ -65,4 +65,3 @@ def __init__(self, key): self.api_key = key self.base_url = 'https://api.shodan.io' self.stream = self.Stream(self) - diff --git a/tests/test_shodan.py b/tests/test_shodan.py index 0cdd602..94ffc70 100644 --- a/tests/test_shodan.py +++ b/tests/test_shodan.py @@ -9,148 +9,174 @@ class ShodanTests(unittest.TestCase): - api = None - FACETS = [ - 'port', - ('domain', 1) - ] - QUERIES = { - 'simple': 'cisco-ios', - 'minify': 'apache', - 'advanced': 'apache port:443', - 'empty': 'asdasdasdasdasdasdasdasdasdhjihjkjk', - } - - def setUp(self): - self.api = shodan.Shodan(open('SHODAN-API-KEY').read().strip()) - - def test_search_simple(self): - results = self.api.search(self.QUERIES['simple']) - - # Make sure the properties exist - self.assertIn('matches', results) - self.assertIn('total', results) - - # Make sure no error occurred - self.assertNotIn('error', results) - - # Make sure some values were returned - self.assertTrue(results['matches']) - self.assertTrue(results['total']) - - # A regular search shouldn't have the optional info - self.assertNotIn('opts', results['matches'][0]) - - def test_search_empty(self): - results = self.api.search(self.QUERIES['empty']) - self.assertTrue(len(results['matches']) == 0) - self.assertEqual(results['total'], 0) - - def test_search_facets(self): - results = self.api.search(self.QUERIES['simple'], facets=self.FACETS) - - self.assertTrue(results['facets']['port']) - self.assertEqual(len(results['facets']['domain']), 1) - - def test_count_simple(self): - results = self.api.count(self.QUERIES['simple']) - - # Make sure the properties exist - self.assertIn('matches', results) - self.assertIn('total', results) - - # Make sure no error occurred - self.assertNotIn('error', results) - - # Make sure no values were returned - self.assertFalse(results['matches']) - self.assertTrue(results['total']) - - def test_count_facets(self): - results = self.api.count(self.QUERIES['simple'], facets=self.FACETS) - - self.assertTrue(results['facets']['port']) - self.assertEqual(len(results['facets']['domain']), 1) - - def test_host_details(self): - host = self.api.host('147.228.101.7') - - self.assertEqual('147.228.101.7', host['ip_str']) - self.assertFalse(isinstance(host['ip'], basestring)) - - def test_search_minify(self): - results = self.api.search(self.QUERIES['minify'], minify=False) - self.assertIn('opts', results['matches'][0]) - - def test_exploits_search(self): - results = self.api.exploits.search('apache') - self.assertIn('matches', results) - self.assertIn('total', results) - self.assertTrue(results['matches']) - - def test_exploits_search_paging(self): - results = self.api.exploits.search('apache', page=1) - match1 = results['matches'][0] - results = self.api.exploits.search('apache', page=2) - match2 = results['matches'][0] - - self.assertNotEqual(match1['_id'], match2['_id']) - - def test_exploits_search_facets(self): - results = self.api.exploits.search('apache', facets=['source', ('author', 1)]) - self.assertIn('facets', results) - self.assertTrue(results['facets']['source']) - self.assertTrue(len(results['facets']['author']) == 1) - - def test_exploits_count(self): - results = self.api.exploits.count('apache') - self.assertIn('matches', results) - self.assertIn('total', results) - self.assertTrue(len(results['matches']) == 0) - - def test_exploits_count_facets(self): - results = self.api.exploits.count('apache', facets=['source', ('author', 1)]) - self.assertEqual(len(results['matches']), 0) - self.assertIn('facets', results) - self.assertTrue(results['facets']['source']) - self.assertTrue(len(results['facets']['author']) == 1) - - # Test error responses - def test_invalid_key(self): - api = shodan.Shodan('garbage') - raised = False - try: - api.search('something') - except shodan.APIError as e: - raised = True - - self.assertTrue(raised) - - def test_invalid_host_ip(self): - raised = False - try: - host = self.api.host('test') - except shodan.APIError as e: - raised = True - - self.assertTrue(raised) - - def test_search_empty_query(self): - raised = False - try: - self.api.search('') - except shodan.APIError as e: - raised = True - self.assertTrue(raised) - - def test_search_advanced_query(self): - # The free API plan can't use filters - raised = False - try: - self.api.search(self.QUERIES['advanced']) - except shodan.APIError as e: - raised = True - self.assertTrue(raised) + api = None + FACETS = [ + 'port', + ('domain', 1) + ] + QUERIES = { + 'simple': 'cisco-ios', + 'minify': 'apache', + 'advanced': 'apache port:443', + 'empty': 'asdasdasdasdasdasdasdasdasdhjihjkjk', + } + + def setUp(self): + with open('SHODAN-API-KEY') as f: + self.api = shodan.Shodan(f.read().strip()) + + def test_search_simple(self): + results = self.api.search(self.QUERIES['simple']) + + # Make sure the properties exist + self.assertIn('matches', results) + self.assertIn('total', results) + + # Make sure no error occurred + self.assertNotIn('error', results) + + # Make sure some values were returned + self.assertTrue(results['matches']) + self.assertTrue(results['total']) + + # A regular search shouldn't have the optional info + self.assertNotIn('opts', results['matches'][0]) + + def test_search_empty(self): + results = self.api.search(self.QUERIES['empty']) + self.assertTrue(len(results['matches']) == 0) + self.assertEqual(results['total'], 0) + + def test_search_facets(self): + results = self.api.search(self.QUERIES['simple'], facets=self.FACETS) + + self.assertTrue(results['facets']['port']) + self.assertEqual(len(results['facets']['domain']), 1) + + def test_count_simple(self): + results = self.api.count(self.QUERIES['simple']) + + # Make sure the properties exist + self.assertIn('matches', results) + self.assertIn('total', results) + + # Make sure no error occurred + self.assertNotIn('error', results) + + # Make sure no values were returned + self.assertFalse(results['matches']) + self.assertTrue(results['total']) + + def test_count_facets(self): + results = self.api.count(self.QUERIES['simple'], facets=self.FACETS) + + self.assertTrue(results['facets']['port']) + self.assertEqual(len(results['facets']['domain']), 1) + + def test_host_details(self): + host = self.api.host('147.228.101.7') + + self.assertEqual('147.228.101.7', host['ip_str']) + self.assertFalse(isinstance(host['ip'], basestring)) + + def test_search_minify(self): + results = self.api.search(self.QUERIES['minify'], minify=False) + self.assertIn('opts', results['matches'][0]) + + def test_exploits_search(self): + results = self.api.exploits.search('apache') + self.assertIn('matches', results) + self.assertIn('total', results) + self.assertTrue(results['matches']) + + def test_exploits_search_paging(self): + results = self.api.exploits.search('apache', page=1) + match1 = results['matches'][0] + results = self.api.exploits.search('apache', page=2) + match2 = results['matches'][0] + + self.assertNotEqual(match1['_id'], match2['_id']) + + def test_exploits_search_facets(self): + results = self.api.exploits.search('apache', facets=['source', ('author', 1)]) + self.assertIn('facets', results) + self.assertTrue(results['facets']['source']) + self.assertTrue(len(results['facets']['author']) == 1) + + def test_exploits_count(self): + results = self.api.exploits.count('apache') + self.assertIn('matches', results) + self.assertIn('total', results) + self.assertTrue(len(results['matches']) == 0) + + def test_exploits_count_facets(self): + results = self.api.exploits.count('apache', facets=['source', ('author', 1)]) + self.assertEqual(len(results['matches']), 0) + self.assertIn('facets', results) + self.assertTrue(results['facets']['source']) + self.assertTrue(len(results['facets']['author']) == 1) + + def test_trends_search(self): + results = self.api.trends.search('apache', facets=[('product', 10)]) + self.assertIn('total', results) + self.assertIn('matches', results) + self.assertIn('facets', results) + self.assertTrue(results['matches']) + self.assertIn('2023-06', [bucket['key'] for bucket in results['facets']['product']]) + + results = self.api.trends.search('apache', facets=[]) + self.assertIn('total', results) + self.assertIn('matches', results) + self.assertNotIn('facets', results) + self.assertTrue(results['matches']) + self.assertIn('2023-06', [match['month'] for match in results['matches']]) + + def test_trends_search_filters(self): + results = self.api.trends.search_filters() + self.assertIn('has_ipv6', results) + self.assertNotIn('http.html', results) + + def test_trends_search_facets(self): + results = self.api.trends.search_facets() + self.assertIn('product', results) + self.assertNotIn('cpe', results) + + # Test error responses + def test_invalid_key(self): + api = shodan.Shodan('garbage') + raised = False + try: + api.search('something') + except shodan.APIError: + raised = True + + self.assertTrue(raised) + + def test_invalid_host_ip(self): + raised = False + try: + self.api.host('test') + except shodan.APIError: + raised = True + + self.assertTrue(raised) + + def test_search_empty_query(self): + raised = False + try: + self.api.search('') + except shodan.APIError: + raised = True + self.assertTrue(raised) + + def test_search_advanced_query(self): + # The free API plan can't use filters + raised = False + try: + self.api.search(self.QUERIES['advanced']) + except shodan.APIError: + raised = True + self.assertTrue(raised) if __name__ == '__main__': diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..1a9f632 --- /dev/null +++ b/tox.ini @@ -0,0 +1,14 @@ +[flake8] +ignore = + E501 W293 + +exclude = + build, + docs, + shodan.egg-info, + tmp, + +per-file-ignores = + shodan/__init__.py:F401, + shodan/cli/converter/__init__.py:F401, + shodan/cli/worldmap.py:W291, \ No newline at end of file