commit c580bbd913e9891e7f4e0b58b86af938773851db Author: Dave G Date: Wed Mar 7 23:57:52 2018 -0500 first commit. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7a03a2a --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +**.egg-info/ +**.pyc +**.pyo +.pypirc +.pytest_cache/ +build/ +dist/ +venv/ +setup.py \ No newline at end of file diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..d31bb65 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,344 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Use multiple processes to speed up Pylint. +jobs=1 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time. See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" + +disable= + attribute-defined-outside-init, + duplicate-code, + fixme, + invalid-name, + missing-docstring, + protected-access, + too-few-public-methods, + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=_$|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=200 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=2000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[BASIC] + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,input + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=__.*__ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# List of decorators that define properties, such as abc.abstractproperty. +property-classes=abc.abstractproperty + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis +ignored-modules= + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). +ignored-classes=SQLObject, optparse.Values, thread._local, _thread._local + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members=REQUEST,acl_users,aq_parent + +# List of decorators that create context managers from functions, such as +# contextlib.contextmanager. +contextmanager-decorators=contextlib.contextmanager + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: none. To make it working +# install python-enchant package. +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=10 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=25 + +# Maximum number of return / yield for function / method body +max-returns=11 + +# Maximum number of branch for function / method body +max-branches=26 + +# Maximum number of statements in function / method body +max-statements=100 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=11 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=25 + + +[CLASSES] + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..6d0e99c --- /dev/null +++ b/.travis.yml @@ -0,0 +1,12 @@ +sudo: false +language: python +python: + - "2.7" + - "3.4" + - "3.5" + - "3.6" + - "3.7-dev" +install: + - pip install tox-travis +script: + - tox \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7670516 --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +Copyright 2018 Dave Gallant + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..485beef --- /dev/null +++ b/README.rst @@ -0,0 +1,17 @@ +RFD CLI +=================== + +.. image:: https://travis-ci.org/davegallant/rfd_cli.svg?branch=prototype + :target: https://travis-ci.org/davegallant/rfd_cli + + +Installation: + + pip install rfd + +Usage + + rfd threads 9 --count 10 + + +.. image:: images/rfd_threads_9.png diff --git a/images/rfd_threads_9.png b/images/rfd_threads_9.png new file mode 100644 index 0000000..5a3cb09 Binary files /dev/null and b/images/rfd_threads_9.png differ diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d29b93d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +beautifulsoup4==4.6.0 +click==6.7 +colorama==0.3.9 +pytest==3.4.2 +requests==2.18.4 \ No newline at end of file diff --git a/rfd/README.md b/rfd/README.md new file mode 100644 index 0000000..e69de29 diff --git a/rfd/__init__.py b/rfd/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/rfd/api.py b/rfd/api.py new file mode 100644 index 0000000..c70d757 --- /dev/null +++ b/rfd/api.py @@ -0,0 +1,102 @@ +"""RFD API.""" + +from math import ceil +import requests +from bs4 import BeautifulSoup + +try: + from urllib.parse import urlparse # python 3 +except ImportError: + from urlparse import urlparse # python 2 + + +def build_web_path(slug): + return "https://forums.redflagdeals.com{}".format(slug) + + +def extract_post_id(url): + return url.split('/')[3].split('-')[-1] + + +def is_int(number): + try: + int(number) + return True + except ValueError: + return False + + +def get_vote_score(up_vote, down_vote): + return up_vote - down_vote + + +def get_safe_per_page(limit): + if limit < 5: + return 5 + elif limit > 40: + return 40 + return limit + + +def users_to_dict(users): + users_dict = {} + for user in users: + users_dict[user.get('user_id')] = user.get('username') + return users_dict + + +def strip_html(text): + return BeautifulSoup(text, "html.parser").get_text() + + +def is_valid_url(url): + result = urlparse(url) + return all([result.scheme, result.netloc, result.path]) + + +def get_threads(forum_id, limit): + threads = [] + response = requests.get( + "https://forums.redflagdeals.com/api/topics?forum_id={}&per_page={}".format(forum_id, get_safe_per_page(limit))) + for topic in response.json().get('topics'): + threads.append({ + 'title': topic.get('title'), + 'score': get_vote_score(topic.get('votes').get('total_up'), topic.get('votes').get('total_down')), + 'url': build_web_path(topic.get('web_path')), + }) + return threads[:limit] + + +def get_posts(post, limit=5): + if is_valid_url(post): + post_id = extract_post_id(post) + elif is_int(post): + post_id = post + else: + raise ValueError() + + if limit == 0: + response = requests.get( + "https://forums.redflagdeals.com/api/topics/{}/posts?per_page=40&page=1".format(post_id)) + pages = response.json().get('pager').get('total_pages') + elif limit > 40: + pages = ceil(limit / 40) + else: + pages = 1 + + # Go through as many pages as necessary + for page in range(pages): + page = page + 1 # page 0 causes issues + response = requests.get( + "https://forums.redflagdeals.com/api/topics/{}/posts?per_page={}&page={}".format(post_id, + get_safe_per_page( + limit), + page)) + + users = users_to_dict(response.json().get('users')) + for _post in response.json().get('posts'): + yield{ + 'body': strip_html(_post.get('body')), + 'score': get_vote_score(_post.get('votes').get('total_up'), _post.get('votes').get('total_down')), + 'user': users[_post.get('author_id')], + } diff --git a/rfd/rfd_cli.py b/rfd/rfd_cli.py new file mode 100644 index 0000000..760d8b8 --- /dev/null +++ b/rfd/rfd_cli.py @@ -0,0 +1,91 @@ +import sys +import os +import click +from colorama import init, Fore, Style +from rfd.api import get_threads, get_posts + +init() +print() + + +def get_terminal_width(): + _, columns = os.popen('stty size', 'r').read().split() + return int(columns) + + +def get_vote_color(score): + if score > 0: + return Fore.GREEN + " [+" + str(score) + "] " + elif score < 0: + return Fore.RED + " [" + str(score) + "] " + return Fore.BLUE + " [" + str(score) + "] " + + +@click.group() +def cli(): + """Welcome to the RFD CLI. (RedFlagDeals.com)""" + pass + + +@cli.command(short_help="Displays posts in a specific thread.") +@click.option('--count', default=5, help='Number of topics. 0 for all topics') +@click.argument('post_id') +def posts(count, post_id): + """Displays posts in a specific thread. + + post_id can be a full url or post id only + + Example: + + \b + url: https://forums.redflagdeals.com/koodo-targeted-public-mobile-12-120-koodo-5gb-40-no-referrals-2173603 + post_id: 2173603 + """ + if count < 0: + click.echo("Invalid count.") + sys.exit(1) + + try: + click.echo("-" * get_terminal_width()) + for post in get_posts(post_id, count): + click.echo(" -" + get_vote_color(post.get('score')) + Fore.RESET + + post.get('body') + Fore.YELLOW + " ({})".format(post.get('user'))) + click.echo(Style.RESET_ALL) + click.echo("-" * get_terminal_width()) + except ValueError: + click.echo("Invalid post id.") + sys.exit(1) + except AttributeError: + click.echo("AttributeError: RFD API did not return expected data.") + + +@cli.command(short_help="Displays threads in the specified forum.") +@click.option('--count', default=5, help='Number of topics.') +@click.argument('forum_id') +def threads(count, forum_id): + """Displays threads in the specified forum id. + + Popular forum ids: + + \b + 9 \t hot deals + 14 \t computer and electronics + 15 \t offtopic + 17 \t entertainment + 18 \t food and drink + 40 \t automotive + 53 \t home and garden + 67 \t fashion and apparel + 74 \t shopping discussion + 88 \t cell phones + """ + _threads = get_threads(forum_id, count) + for i, thread in enumerate(_threads, 1): + click.echo(" " + str(i) + "." + + get_vote_color(thread.get('score')) + Fore.RESET + thread.get('title')) + click.echo(Fore.BLUE + " {}".format(thread.get('url'))) + click.echo(Style.RESET_ALL) + + +if __name__ == '__main__': + cli() diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..4767e00 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1 @@ +pylint==1.8.2 \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..e8d3579 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,10 @@ +from rfd.api import build_web_path, extract_post_id + + +def test_build_web_path(): + assert build_web_path("/test") == "https://forums.redflagdeals.com/test" + + +def test_extract_post_id(): + assert extract_post_id("https://forums.redflagdeals.com/targeted-bob-2173603/120") == '2173603' + assert extract_post_id("http://forums.redflagdeals.com/targeted-2173604/120") == '2173604' diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..597ac0b --- /dev/null +++ b/tox.ini @@ -0,0 +1,11 @@ +[tox] +envlist = py{27,34,35,36,37} + +[testenv] +commands = + py.test + pylint rfd + +deps = + -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt \ No newline at end of file