websucker.py 3.35 KB
Newer Older
1 2 3 4
#! /usr/bin/env python

"""A variant on webchecker that creates a mirror copy of a remote site."""

5
__version__ = "$Revision$"
6 7 8 9 10 11 12

import os
import sys
import string
import urllib
import getopt

13
import webchecker
14 15 16 17 18

# Extract real version number if necessary
if __version__[0] == '$':
    _v = string.split(__version__)
    if len(_v) == 3:
Guido van Rossum's avatar
Guido van Rossum committed
19
        __version__ = _v[1]
20 21

def main():
22
    verbose = webchecker.VERBOSE
23
    try:
Guido van Rossum's avatar
Guido van Rossum committed
24
        opts, args = getopt.getopt(sys.argv[1:], "qv")
25
    except getopt.error, msg:
Guido van Rossum's avatar
Guido van Rossum committed
26 27 28
        print msg
        print "usage:", sys.argv[0], "[-qv] ... [rooturl] ..."
        return 2
29
    for o, a in opts:
Guido van Rossum's avatar
Guido van Rossum committed
30 31 32 33
        if o == "-q":
            verbose = 0
        if o == "-v":
            verbose = verbose + 1
34 35
    c = Sucker()
    c.setflags(verbose=verbose)
36
    c.urlopener.addheaders = [
Guido van Rossum's avatar
Guido van Rossum committed
37 38
            ('User-agent', 'websucker/%s' % __version__),
        ]
39
    for arg in args:
Guido van Rossum's avatar
Guido van Rossum committed
40 41
        print "Adding root", arg
        c.addroot(arg)
42 43 44 45 46
    print "Run..."
    c.run()

class Sucker(webchecker.Checker):

47
    checkext = 0
Guido van Rossum's avatar
Guido van Rossum committed
48 49 50 51 52
    nonames = 1

    # SAM 11/13/99: in general, URLs are now URL pairs.
    # Since we've suppressed name anchor checking,
    # we can ignore the second dimension.
53

Guido van Rossum's avatar
Guido van Rossum committed
54 55
    def readhtml(self, url_pair):
        url = url_pair[0]
Guido van Rossum's avatar
Guido van Rossum committed
56 57 58 59 60
        text = None
        path = self.savefilename(url)
        try:
            f = open(path, "rb")
        except IOError:
Guido van Rossum's avatar
Guido van Rossum committed
61
            f = self.openpage(url_pair)
Guido van Rossum's avatar
Guido van Rossum committed
62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77
            if f:
                info = f.info()
                nurl = f.geturl()
                if nurl != url:
                    url = nurl
                    path = self.savefilename(url)
                text = f.read()
                f.close()
                self.savefile(text, path)
                if not self.checkforhtml(info, url):
                    text = None
        else:
            if self.checkforhtml({}, url):
                text = f.read()
            f.close()
        return text, url
78 79

    def savefile(self, text, path):
Guido van Rossum's avatar
Guido van Rossum committed
80 81
        dir, base = os.path.split(path)
        makedirs(dir)
82 83 84 85 86 87 88
        try:
            f = open(path, "wb")
            f.write(text)
            f.close()
            self.message("saved %s", path)
        except IOError, msg:
            self.message("didn't save %s: %s", path, str(msg))
89 90

    def savefilename(self, url):
Guido van Rossum's avatar
Guido van Rossum committed
91 92 93 94 95 96
        type, rest = urllib.splittype(url)
        host, path = urllib.splithost(rest)
        while path[:1] == "/": path = path[1:]
        user, host = urllib.splituser(host)
        host, port = urllib.splitnport(host)
        host = string.lower(host)
97
        if not path or path[-1] == "/":
98
            path = path + "index.html"
Guido van Rossum's avatar
Guido van Rossum committed
99 100
        if os.sep != "/":
            path = string.join(string.split(path, "/"), os.sep)
101 102
            if os.name == "mac":
                path = os.sep + path
103
        path = os.path.join(host, path)
Guido van Rossum's avatar
Guido van Rossum committed
104
        return path
105 106

def makedirs(dir):
107 108 109 110 111 112 113 114 115 116
    if not dir:
        return
    if os.path.exists(dir):
        if not os.path.isdir(dir):
            try:
                os.rename(dir, dir + ".bak")
                os.mkdir(dir)
                os.rename(dir + ".bak", os.path.join(dir, "index.html"))
            except os.error:
                pass
Guido van Rossum's avatar
Guido van Rossum committed
117
        return
118 119
    head, tail = os.path.split(dir)
    if not tail:
Guido van Rossum's avatar
Guido van Rossum committed
120 121
        print "Huh?  Don't know how to make dir", dir
        return
122 123 124 125 126
    makedirs(head)
    os.mkdir(dir, 0777)

if __name__ == '__main__':
    sys.exit(main() or 0)