flexget.plugins.input.tail
Covered: 47 lines
Missed: 80 lines
Skipped 43 lines
Percent: 37 %
  1
import os
  2
from flexget.entry import Entry
  3
from flexget.plugin import register_plugin, register_parser_option, get_plugin_by_name, DependencyError, PluginError
  4
from flexget.utils.cached_input import cached
  5
import re
  6
import logging
  7
import sys
  9
log = logging.getLogger('tail')
 12
class ResetTail(object):
 13
    """Adds --tail-reset"""
 15
    def on_process_start(self, feed):
 16
        if not feed.manager.options.tail_reset:
 17
            return
 19
        feed.manager.disable_feeds()
 21
        from flexget.utils.simple_persistence import SimpleKeyValue
 22
        from flexget.manager import Session
 24
        session = Session()
 25
        try:
 26
            poses = session.query(SimpleKeyValue).filter(SimpleKeyValue.key == feed.manager.options.tail_reset).all()
 27
            if not poses:
 28
                print 'No position stored for file %s' % feed.manager.options.tail_reset
 29
                print 'Note that file must give in same format as in config, ie. ~/logs/log can not be given as /home/user/logs/log'
 30
            for pos in poses:
 31
                if pos.value == 0:
 32
                    print 'Feed %s tail position is already zero' % pos.feed
 33
                else:
 34
                    print 'Feed %s tail position (%s) reseted to zero' % (pos.feed, pos.value)
 35
                    pos.value = 0
 36
            session.commit()
 37
        finally:
 38
            session.close()
 41
class InputTail(object):
 43
    """
 44
    Parse any text for entries using regular expression.
 46
    file: <file>
 47
    entry:
 48
      <field>: <regexp to match value>
 49
    format:
 50
      <field>: <python string formatting>
 52
    Note: each entry must have atleast two fields, title and url
 54
    You may wish to specify encoding used by file so file can be properly
 55
    decoded. List of encodings
 56
    at http://docs.python.org/library/codecs.html#standard-encodings.
 58
    Example:
 60
    tail:
 61
      file: ~/irclogs/some/log
 62
      entry:
 63
        title: 'TITLE: (.*) URL:'
 64
        url: 'URL: (.*)'
 65
      encoding: utf8
 66
    """
 68
    def validator(self):
 69
        from flexget import validator
 70
        root = validator.factory('dict')
 71
        root.accept('file', key='file', required=True)
 72
        root.accept('text', key='encoding')
 73
        entry = root.accept('dict', key='entry', required=True)
 74
        entry.accept('regexp', key='url', required=True)
 75
        entry.accept('regexp', key='title', required=True)
 76
        entry.accept_any_key('regexp')
 77
        format = root.accept('dict', key='format')
 78
        format.accept_any_key('text')
 79
        return root
 81
    def format_entry(self, entry, d):
 82
        for k, v in d.iteritems():
 83
            entry[k] = v % entry
 85
    @cached('tail')
 86
    def on_feed_input(self, feed):
 88
        try:
 90
            details = get_plugin_by_name('details').instance
 91
            if feed.name not in details.no_entries_ok:
 92
                log.debug('appending %s to details plugin no_entries_ok' % feed.name)
 93
                details.no_entries_ok.append(feed.name)
 94
        except DependencyError:
 95
            log.debug('unable to get details plugin')
 97
        filename = os.path.expanduser(feed.config['tail']['file'])
 98
        encoding = feed.config['tail'].get('encoding', None)
 99
        file = open(filename, 'r')
101
        last_pos = feed.simple_persistence.setdefault(filename, 0)
102
        if os.path.getsize(filename) < last_pos:
103
            log.info('File size is smaller than in previous execution, reseting to beginning of the file')
104
            last_pos = 0
106
        file.seek(last_pos)
108
        log.debug('continuing from last position %s' % last_pos)
110
        entry_config = feed.config['tail'].get('entry')
111
        format_config = feed.config['tail'].get('format', {})
114
        used = {}
115
        entry = Entry()
119
        while True:
120
            line = file.readline()
121
            if encoding:
122
                try:
123
                    line = line.decode(encoding)
124
                except UnicodeError:
125
                    raise PluginError('Failed to decode file using %s. Check encoding.' % encoding)
127
            if not line:
128
                feed.simple_persistence[filename] = file.tell()
129
                break
131
            for field, regexp in entry_config.iteritems():
133
                match = re.search(regexp, line)
134
                if match:
136
                    if used.has_key(field):
137
                        if entry.isvalid():
138
                            log.info('Found field %s again before entry was completed. \
139
                                      Adding current incomplete, but valid entry and moving to next.' % field)
140
                            self.format_entry(entry, format_config)
141
                            feed.entries.append(entry)
142
                        else:
143
                            log.info('Invalid data, entry field %s is already found once. Ignoring entry.' % field)
145
                        entry = Entry()
146
                        used = {}
149
                    entry[field] = match.group(1)
150
                    used[field] = True
151
                    log.debug('found field: %s value: %s' % (field, entry[field]))
154
                if len(used) == len(entry_config):
156
                    if not entry.isvalid():
157
                        log.info('Invalid data, constructed entry is missing mandatory fields (title or url)')
158
                    else:
159
                        self.format_entry(entry, format_config)
160
                        feed.entries.append(entry)
161
                        log.debug('Added entry %s' % entry)
163
                        entry = Entry()
164
                        used = {}
166
register_plugin(InputTail, 'tail')
167
register_plugin(ResetTail, '--tail-reset', builtin=True)
168
register_parser_option('--tail-reset', action='store', dest='tail_reset', default=False, metavar='FILE',
169
    help='Reset tail position for a file.')