flexget.plugins.input_tail
Covered: 39 lines
Missed: 76 lines
Skipped 40 lines
Percent: 33 %
  1
import os
  2
from flexget.feed import Entry
  3
from flexget.plugin import register_plugin, register_parser_option, get_plugin_by_name, DependencyError
  4
from flexget.utils.cached_input import cached
  5
import re
  6
import logging
  8
log = logging.getLogger('tail')
 11
class ResetTail(object):
 12
    """Adds --tail-reset"""
 14
    def on_process_start(self, feed):
 15
        if not feed.manager.options.tail_reset:
 16
            return
 18
        feed.manager.disable_feeds()
 20
        from flexget.utils.simple_persistence import SimpleKeyValue
 21
        from flexget.manager import Session
 23
        session = Session()
 24
        try:
 25
            poses = session.query(SimpleKeyValue).filter(SimpleKeyValue.key == feed.manager.options.tail_reset).all()
 26
            if not poses:
 27
                print 'No position stored for file %s' % feed.manager.options.tail_reset
 28
                print 'Note that file must give in same format as in config, ie. ~/logs/log can not be given as /home/user/logs/log'
 29
            for pos in poses:
 30
                if pos.value == 0:
 31
                    print 'Feed %s tail position is already zero' % pos.feed
 32
                else:
 33
                    print 'Feed %s tail position (%s) reseted to zero' % (pos.feed, pos.value)
 34
                    pos.value = 0
 35
            session.commit()
 36
        finally:
 37
            session.close()
 40
class InputTail(object):
 42
    """
 43
    Parse any text for entries using regular expression.
 45
    file: <file>
 46
    entry:
 47
      <field>: <regexp to match value>
 48
    format:
 49
      <field>: <python string formatting>
 51
    Note: each entry must have atleast two fields, title and url
 53
    Example:
 55
    tail:
 56
      file: ~/irclogs/some/log
 57
      entry:
 58
        title: 'TITLE: (.*) URL:'
 59
        url: 'URL: (.*)'
 60
    """
 62
    def validator(self):
 63
        from flexget import validator
 64
        root = validator.factory('dict')
 65
        root.accept('file', key='file', required=True)
 66
        entry = root.accept('dict', key='entry', required=True)
 67
        entry.accept('regexp', key='url', required=True)
 68
        entry.accept('regexp', key='title', required=True)
 69
        entry.accept_any_key('regexp')
 70
        format = root.accept('dict', key='format')
 71
        format.accept_any_key('text')
 72
        return root
 74
    def format_entry(self, entry, d):
 75
        for k, v in d.iteritems():
 76
            entry[k] = v % entry
 78
    @cached('tail', 'file')
 79
    def on_feed_input(self, feed):
 81
        try:
 83
            details = get_plugin_by_name('details').instance
 84
            if feed.name not in details.no_entries_ok:
 85
                log.debug('appending %s to details plugin no_entries_ok' % feed.name)
 86
                details.no_entries_ok.append(feed.name)
 87
        except DependencyError:
 88
            log.debug('unable to get details plugin')
 90
        filename = os.path.expanduser(feed.config['tail']['file'])
 91
        file = open(filename, 'r')
 93
        last_pos = feed.simple_persistence.setdefault(filename, 0)
 94
        if os.path.getsize(filename) < last_pos:
 95
            log.info('File size is smaller than in previous execution, reseting to beginning of the file')
 96
            last_pos = 0
 98
        file.seek(last_pos)
100
        log.debug('continuing from last position %s' % last_pos)
102
        entry_config = feed.config['tail'].get('entry')
103
        format_config = feed.config['tail'].get('format', {})
106
        used = {}
107
        entry = Entry()
110
        while True:
111
            line = file.readline()
112
            if not line:
113
                feed.simple_persistence.set(filename, file.tell())
114
                break
116
            for field, regexp in entry_config.iteritems():
118
                match = re.search(regexp, line)
119
                if match:
121
                    if used.has_key(field):
122
                        if entry.isvalid():
123
                            log.info('Found field %s again before entry was completed. \
124
                                      Adding current incomplete, but valid entry and moving to next.' % field)
125
                            self.format_entry(entry, format_config)
126
                            feed.entries.append(entry)
127
                        else:
128
                            log.info('Invalid data, entry field %s is already found once. Ignoring entry.' % field)
130
                        entry = Entry()
131
                        used = {}
134
                    entry[field] = match.group(1)
135
                    used[field] = True
136
                    log.debug('found field: %s value: %s' % (field, entry[field]))
139
                if len(used) == len(entry_config):
141
                    if not entry.isvalid():
142
                        log.info('Invalid data, constructed entry is missing mandatory fields (title or url)')
143
                    else:
144
                        self.format_entry(entry, format_config)
145
                        feed.entries.append(entry)
146
                        log.debug('Added entry %s' % entry)
148
                        entry = Entry()
149
                        used = {}
151
register_plugin(InputTail, 'tail')
152
register_plugin(ResetTail, '--tail-reset', builtin=True)
153
register_parser_option('--tail-reset', action='store', dest='tail_reset', default=False, metavar='FILE',
154
    help='Reset tail position for a file.')