flexget.plugins.input.discover
Covered: 59 lines
Missed: 47 lines
Skipped 20 lines
Percent: 55 %
  1
import logging

  2
from flexget.utils.cached_input import cached

  3
from flexget.utils.search import StringComparator, MovieComparator, AnyComparator, clean_title

  4
from flexget.plugin import register_plugin, get_plugin_by_name, PluginError, \

  5
    get_plugins_by_group, get_plugins_by_phase, PluginWarning

  7
log = logging.getLogger('discover')

 10
class Discover(object):

 11
    """ Discover content based on other inputs material.

 13
    Example:

 15
    discover:

 16
      what:

 17
        - emit_series: yes

 18
      from:

 19
        - piratebay

 20
    """

 22
    def validator(self):

 23
        from flexget import validator

 24
        discover = validator.factory('dict')

 26
        inputs = discover.accept('list', key='what', required=True).accept('dict')

 27
        for plugin in get_plugins_by_phase('input'):

 28
            if hasattr(plugin.instance, 'validator'):

 29
                inputs.accept(plugin.instance.validator, key=plugin.name)

 31
        searches = discover.accept('list', key='from', required=True)

 32
        no_config = searches.accept('choice')

 33
        for plugin in get_plugins_by_group('search'):

 34
            if hasattr(plugin.instance, 'validator'):

 35
                searches.accept('dict').accept(plugin.instance.validator, key=plugin.name)

 36
            else:

 37
                no_config.accept(plugin.name)

 39
        discover.accept('integer', key='limit')

 40
        discover.accept('choice', key='type').accept_choices(['any', 'normal', 'exact', 'movies'])

 41
        return discover

 43
    def execute_inputs(self, config, feed):

 44
        """

 45
        :param config: Discover config

 46
        :param feed: Current feed

 47
        :return: List of pseudo entries created by inputs under `what` configuration

 48
        """

 50
        entries = []

 51
        entry_titles = set()

 52
        entry_urls = set()

 54
        for item in config['what']:

 55
            for input_name, input_config in item.iteritems():

 56
                input = get_plugin_by_name(input_name)

 57
                if input.api_ver == 1:

 58
                    raise PluginError('Plugin %s does not support API v2' % input_name)

 59
                method = input.phase_handlers['input']

 60
                try:

 61
                    result = method(feed, input_config)

 62
                except PluginError, e:

 63
                    log.warning('Error during input plugin %s: %s' % (input_name, e))

 64
                    continue

 65
                if not result:

 66
                    log.warning('Input %s did not return anything' % input_name)

 67
                    continue

 69
                for entry in result:

 70
                    urls = ([entry['url']] if entry.get('url') else []) + entry.get('urls', [])

 71
                    if any(url in entry_urls for url in urls):

 72
                        log.debug('URL for `%s` already in entry list, skipping.' % entry['title'])

 73
                        continue

 75
                    if entry['title'] in entry_titles:

 76
                        log.verbose('Ignored duplicate title `%s`' % entry['title']) # TODO: should combine?

 77
                    else:

 78
                        entries.append(entry)

 79
                        entry_titles.add(entry['title'])

 80
                        entry_urls.update(urls)

 81
        return entries

 83
    def execute_searches(self, config, entries):

 84
        """

 85
        :param config: Discover plugin config

 86
        :param entries: List of pseudo entries to search

 87
        :return: List of entries found from search engines listed under `from` configuration

 88
        """

 90
        result = []

 91
        if config.get('type', 'normal') == 'normal':

 92
            comparator = StringComparator(cutoff=0.7, cleaner=clean_title)

 93
        elif config['type'] == 'exact':

 94
            comparator = StringComparator(cutoff=0.9)

 95
        elif config['type'] == 'any':

 96
            comparator = AnyComparator()

 97
        else:

 98
            comparator = MovieComparator()

 99
        for item in config['from']:

100
            if isinstance(item, dict):

101
                plugin_name, plugin_config = item.items()[0]

102
            else:

103
                plugin_name, plugin_config = item, None

104
            search = get_plugin_by_name(plugin_name).instance

105
            if not callable(getattr(search, 'search')):

106
                log.critical('Search plugin %s does not implement search method' % plugin_name)

107
            for entry in entries:

108
                try:

109
                    search_results = search.search(entry['title'], comparator, plugin_config)

110
                    log.debug('Discovered %s entries from %s' % (len(search_results), plugin_name))

111
                    result.extend(search_results[:config.get('limit')])

112
                except (PluginError, PluginWarning):

113
                    log.debug('No results from %s' % plugin_name)

114
        return sorted(result, reverse=True, key=lambda x: x.get('search_sort'))

116
    @cached('discover')

117
    def on_feed_input(self, feed, config):

118
        entries = self.execute_inputs(config, feed)

119
        log.verbose('Discovering %i titles ...' % len(entries))

120
        if len(entries) > 500:

121
            log.critical('Looks like your inputs in discover configuration produced over 500 entries, please reduce the amount!')

122
        return self.execute_searches(config, entries)

125
register_plugin(Discover, 'discover', api_ver=2)