{{{~cpp
#!/usr/bin/env python

# Copyright (c) 2005, Google Inc.
# All rights reserved.
# 
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
# 
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
# 
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import robotparser
import urllib
import urllib2
import re
import sets
import sys
import urlparse
import win32com.client
import time
import pywintypes
import pythoncom
import optparse
import getpass
import itertools
import email.Utils

'''A simple web crawler that pushes pages into GDS.  Features include:
  - Knows basic and digest HTTP authentication
  - Obeys robots.txt
  - Can loop, recrawling over previously crawled pages every X minutes
  - When recrawling, uses If-Modified-Since HTTP header to minimize transfers

For usage instructions, run with -h flag.

Requires Python 2.4 and the win32all extensions for Python 2.4 on Windows.
Will not work unless Google Desktop Search 1.0 or later is installed.
'''

# Matches URLs in <a href=...> tags.  Chosen above htmllib.HTMLParser because
# this is much more lenient, not requiring HTML to be valid.
_LINK_RE = re.compile(r'<\s*(a|img).+href\s*=\s*"?(.+?)"?(\s|>)',
                      re.MULTILINE | re.IGNORECASE)


# Matches <frame src="bla"> tags.
_FRAME_RE = re.compile(r'<\s*(frame).+src\s*=\s*"?(.+?)"?(\s|>)',
                       re.MULTILINE | re.IGNORECASE)


# Digs out the text of an HTML document's title.
_TITLE_RE = re.compile(r'<\s*title.*?>(.+)</\s*title\s*>',
                       re.MULTILINE | re.IGNORECASE)


# This plugin's GUID, used to register with GDS.
_GUID = '{5e1788fe-a6e6-429f-816c-80cb969028d3}'


class NoExceptionHandler(urllib2.BaseHandler):
  '''An exception handler for HTTP that never throws an exception for various
  error codes that Kongulo always checks explicitly rather than catching them
  as exceptions.'''
  def http_error_304(self, req, fp, code, msg, hdrs):
    '''We handle not-modified-since explicitly.'''
    return fp
  
  # We check error codes explicitly so we don't want an exception
  http_error_400 = http_error_401 = http_error_402 = http_error_403 \
  = http_error_404 = http_error_304


class PasswordDb(urllib2.HTTPPasswordMgr):
  '''A very simple password store.  The user can supply usernames using the
  -p flag on the command line, and will be prompted for the password for
  each username.'''
  
  def __init__(self):
    self.passwords = []  # [ [substring, uid, pw], [substring, uid, pw] ]
  
  def Populate(self, options):
    '''Given an options object as used by Kongulo, ask the user for the
    password for each user-id/substring-of-domain that the user provided using
    the -p flag.'''
    if not options.pw:
      return
    
    for item in options.pw.split(','):
      (uid, substring) = item.split('@')
      pw = getpass.getpass('Enter password for %s: ' % item)
      self.passwords.append([substring, uid, pw])
    
  def find_user_password(self, *args, **kw):
    for passdata in self.passwords:
      for name in args:
        if name.find(passdata[0]) != -1:
          return (passdata[1], passdata[2])
    print "!!! Need login info for (%s @ %s), consider using -p flag" % args
    return (None, None)

passwords = PasswordDb()

# A URL opener that can do basic and digest authentication, and never raises
# exceptions for HTTP error codes we handle explicitly.
opener = urllib2.build_opener(urllib2.HTTPBasicAuthHandler(passwords),
                              urllib2.HTTPDigestAuthHandler(passwords),
                              NoExceptionHandler())

# To be a nice Internet citizen, we identify ourselves properly so that
# whoever doesn't like Kongulo can exclude us using robots.txt
opener.addheaders = [('User-agent', 'Kongulo v0.1 personal web crawler')]

# Should always be true on Windows systems.
assert hasattr(opener.handlers[0],
               'proxies'), 'ProxyHandler must be first handler.'
# This parses Windows proxy registry settings
opener.handlers[0].proxies = urllib.getproxies()

class LenientRobotParser(robotparser.RobotFileParser):
  '''Adds ability to parse robot files where same user agent is specified
  multiple times.'''

  def __init__(self, url):
    '''Setup internal state like RobotFileParser does.'''
    robotparser.RobotFileParser.__init__(self)
    f = opener.open(url)
    lines = []
    line = f.readline()
    while line:
        lines.append(line.strip())
        line = f.readline()
    self.errcode = f.code
    if self.errcode == 401 or self.errcode == 403:
        self.disallow_all = 1
    elif self.errcode >= 400:
        self.allow_all = 1
    elif self.errcode == 200 and lines:
        self.parse(lines)
    
  def parse(self, lines):
    """Strip repeated sequential definitions of same user agent, then
    call base's parse method."""
    last_ua = ''
    modified_lines = []
    for line in lines:
      line
      if line.lower().startswith('user-agent'):
        temp = last_ua
        last_ua = line.lower()
        if last_ua == temp:
          continue  # skip line
      if line.strip() == '':
        last_ua = ''  # reset on blank line
      modified_lines += [line]
    
    robotparser.RobotFileParser.parse(self, modified_lines)


class UrlValidator:
  '''An object that handles checking if we should fetch and crawl a specific
  URL.  This is based on the type of the URL (only crawl http URLs) and robot
  rules.  Maintains a cache of robot rules already fetched.'''
  
  def __init__(self, match_url):
    self.robots = {}  # Dict of robot URLs to robot parsers
    self.match_url = re.compile(match_url)
  
  def IsCrawlable(self, url):
    """Returns true if it's OK to crawl the absolute URL provided."""
    if not url.startswith('http') or not self.match_url.match(url):
      return 0
    return self.GetRules(url).can_fetch('*', url)
  
  def GetRules(self, url):
    """Returns the robot rules parser for 'url'"""
    robots_dir = urlparse.urljoin(url, "robots.txt")  # First try dir-level
    if robots_dir in self.robots:
      return self.robots[robots_dir]
    robots_site = urlparse.urljoin(url, "/robots.txt")  # Then the site-level
    if robots_site in self.robots:
      return self.robots[robots_site]
    
    # Inv: Our cache contains neither a dir-level nor site-level robots.txt file
    
    rules = LenientRobotParser(robots_dir)  # First try dir-level
    if hasattr(rules, 'errcode') and rules.errcode == 200:
      self.robots[robots_dir] = rules
    else:
      rules = LenientRobotParser(robots_site)  # Then try site-level
      self.robots[robots_site] = rules
    
    return rules


class Crawler:
  '''This object holds the state of the crawl, and performs the crawl.'''
  
  def __init__(self, options):    
    self.options = options  # Store the options provided
    self.rules = UrlValidator(options.match)  # Cache of robot rules etc.
    
    # Invariant of data:
    # - 'tocrawl' is a list of items that we have or will crawl.  If we have
    #   never crawled them since we started, the item at index 2 in each
    #   crawlitem is None, otherwise it is a dictionary of headers,
    #   specifically the 'If-Modified-Since' header, to prevent us from fetching
    #   this item in the next crawl if it hasn't been modified.
    # - 'scheduled' is a list of items we have already added to 'tocrawl'
    #   (perhaps a premature optimization since we could just iterate over
    #   'tocrawl')
    self.scheduled = sets.Set()
    # Format of this list is:
    # [[url1, depth1, { headername : headerval, ... } ], [url2, depth2], {}...]
    self.tocrawl = []
    
    # Fetch the entrypoint to the Google Desktop Search API.

  
  def ExtractLinks(self, baseurl, htmldoc):
    """Returns all anchors from the document with contents 'htmldoc' at
    'baseurl' that are OK to crawl."""
    urls = []
    for match in itertools.chain(_LINK_RE.finditer(htmldoc),
                                 _FRAME_RE.finditer(htmldoc)):
      url = urlparse.urljoin(baseurl, match.group(2))
      if self.rules.IsCrawlable(url):
        urls += [url]
      else:
        print "    I %s" % url
    return urls
  
  def Crawl(self, baseurls):
    '''Performs the crawl.
    
    Args:
      baseurls: [url1, url2, ...]
    '''
    # Bootstrap our invariant of data
    for baseurl in baseurls:
      self.tocrawl.append([baseurl, self.options.depth, None])
    
    if self.options.loop:
      print "Running in loop mode - press Ctrl-C to stop."
    
    while True:
      for crawlitem in self.tocrawl:
        (url, depth, headers) = crawlitem
        try:
          if headers:
            doc = opener.open(urllib2.Request(url, headers=headers))
          else:
            doc = opener.open(url)
          
          doctype = doc.info().type
          if doc.code == 304:  # not modified since last time
            print "--- (nomod) %s" % url
          elif (doc.code == 200 and doctype == 'text/html' or
                doctype == 'text/plain'):
            print "::: (%d) %s" % (depth, url)
            
            # Store last modified in the crawlitem
            # Prefer Last-Modified header, then Date header (to get same
            # formatting as used by the server), then current date in
            # appropriate format.
            last_modified = None
            if 'last_modified' in doc.headers:
              last_modified = fdoc.headers['last_modified']
            elif 'date' in doc.headers:
              last_modified = doc.headers['date']
            else:
              last_modified = email.Utils.formatdate(time.time(), usegmt=True)
            crawlitem[2] = { 'If-Modified-Since' : last_modified }
            
            content = doc.read()
            
            # Create a GDS event, populate its fields, and send it off to have
            # the web page added to the Google Desktop Search index.
            
            #event.AddProperty('format', doctype)
            #event.AddProperty('content', content)
            #event.AddProperty('uri', url)
            fout = file("output.txt",'w');
            print >> fout,"\n****doctype********\n" 
            print >> fout, doctype
            print >> fout,"\n****content********\n"
            print >> fout, content
            print >> fout,"\n******url******\n"
            print >> fout, url

            # TODO Use the last-modified HTTP header instead of current time
            # if available.
            #event.AddProperty('last_modified_time',
            #                  pywintypes.Time(time.time() + time.timezone))
            print >> fout,"\n*****time*********\n"            
            print >> fout, pywintypes.Time(time.time() + time.timezone)

            
            if doctype == 'text/html':  # no links in text documents
              title_match = _TITLE_RE.search(content)
              if title_match:
                title = title_match.group(1)
                #event.AddProperty('title', title)
                print >> fout, "\n*****title*******\n"
                print >> fout, title
              
              for link in self.ExtractLinks(doc.geturl(), content):
                if depth > 0 and not link in self.scheduled:
                  self.scheduled.add(link)
                  self.tocrawl.append([link, depth - 1, None])
  
            # Don't use historical flag, because if we do, GDS will "throttle"
            # the events we send, not returning until the user becomes idle.
            # We also want to ensure the page is updated in the cache (in case
            # the user already visited it herself using a browser).
            #event.Send(0x01)
          else:
            print "!!! (HTTP %d) %s" % (doc.code, url)
    
          doc.close()
        except IOError:
          print "!!! (nolink) %s" % url
        except ValueError:
          print "!!! (noauth) %s" % url
      
      if not self.options.loop:
        break
      else:
        print ("=== Completed crawl; will recrawl in %d minutes." %
               (self.options.sleep))
        time.sleep(60 * self.options.sleep)


def Main():
  '''This function contains the logic for the command-line UI for Kongulo.'''
  
  # Set up options and parse arguments.
  parser = optparse.OptionParser(usage='%prog [options] BASEURL1 BASEURL2 ...')
  parser.add_option('-d', '--depth', type='int', dest='depth', default=0,
                    help='How deep to follow links from BASEURLs (default 0, '
                         'suggest max 5-6)')
  parser.add_option('-m', '--match', dest='match', default='.+',
                    help=r'Regular expression that URLs must match if they are '
                    'to be crawled, e.g. ".+intranet\.smurfgeburf\.com.+" to '
                    'stay within the Smurfgeburf intranet')
  parser.add_option('-l', '--loop', action='store_true', dest='loop',
                    default=False, help='If this flag is given, Kongulo will '
                    'keep fetching the specified page and pages it points to.  '
                    'It will not refetch pages that haven't changed.')
  parser.add_option('-s', '--sleep', type='int', dest='sleep', default=60,
                    help='Number of minutes to sleep before looping (default '
                    '60). Only valid if -l is also specified.')
  parser.add_option('-p', '--passwords', dest='pw',
                    help='Comma-delimited list of user IDs at names that will '
                    'be matched as substrings against the domain or "region" '
                    'that a password is needed for, e.g. '
                    '"joi@google.com,admin@192.168.250.1,snafu@slashdot.org".  '
                    'You will be prompted for each password.')
  (options, args) = parser.parse_args()
  if len(args) < 1:
    parser.error('Provide at least one base URL')
  
#  try:
#    obj = win32com.client.Dispatch('GoogleDesktopSearch.Register')
#  except pythoncom.ole_error:
#    print ('ERROR: You need to install Google Desktop Search to be able to '
#           'use Kongulo.')
 #   sys.exit(2)
  
#  try:
    # Register with GDS.  This is a one-time operation and will return an
    # error if already registered.  We cheat and just catch the error and
    # do nothing.
 #   obj.RegisterComponent(_GUID,
 #            ['Title', 'Kongulo', 'Description', 'A simple web spider that '
 #             'lets you keep copies of web sites in your Google Desktop Search '
 #             'index.', 'Icon', '%SystemRoot%\system32\SHELL32.dll,134'])
    
    # TODO Provide an unregistration mechanism.
 # except:
    # TODO narrow to only the error that GDS returns when component
    # already registered
 #   pass

  passwords.Populate(options)
  Crawler(options).Crawl(args)


if __name__ == '__main__':
  Main()
}}}