wikiteam/gui.py

#!/usr/bin/env python2
# -*- coding: utf-8 -*-

# Copyright (C) 2011-2012 WikiTeam
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import platform
import random
import re
from Tkinter import *
import ttk
import tkMessageBox
import thread
import time
import urllib
import webbrowser

import dumpgenerator

"""
TODO:

* basic: GUI to download just a wiki

* advanced: batch downloads, upload to Internet Archive or anywhere
"""

wikifarms = {
    'gentoo_wikicom': 'Gentoo Wiki',
    'opensuseorg': 'OpenSuSE',
    'referatacom': 'Referata',
    'shoutwikicom': 'ShoutWiki',
    'Unknown': 'Unknown',
    'wikanda': 'Wikanda',
    'wikifur': 'WikiFur',
    'wikimedia': 'Wikimedia',
    'wikitravelorg': 'WikiTravel',
    'wikkii': 'Wikkii',
}

NAME = 'WikiTeam tools'
VERSION = '0.1'
HOMEPAGE = 'https://code.google.com/p/wikiteam/'
LINUX = platform.system().lower() == 'linux'
PATH = os.path.dirname(__file__)
if PATH: os.chdir(PATH)

class App:
    def __init__(self, master):
        self.master = master
        self.dumps = []
        self.downloadpath = 'downloads'
        self.block = False
        
        # interface elements
        #progressbar
        #self.value = 0
        #self.progressbar = ttk.Progressbar(self.master, orient=HORIZONTAL, value=self.value, mode='determinate')
        #self.progressbar.grid(row=0, column=0, columnspan=1, sticky=W+E)
        #self.run()
        
        #description
        self.desc = Label(self.master, text="", anchor=W, font=("Arial", 10))
        self.desc.grid(row=0, column=0, columnspan=1)
        #self.footer = Label(self.master, text="%s (version %s). This program is free software (GPL v3 or higher)" % (NAME, VERSION), anchor=W, justify=LEFT, font=("Arial", 10))
        #self.footer.grid(row=2, column=0, columnspan=1)
        
        #begin tabs
        self.notebook = ttk.Notebook(self.master)
        self.notebook.grid(row=1, column=0, columnspan=1, sticky=W+E+N+S)
        self.frame1 = ttk.Frame(self.master)
        self.notebook.add(self.frame1, text='Dump generator')
        self.frame2 = ttk.Frame(self.master)
        self.notebook.add(self.frame2, text='Downloader')
        self.frame3 = ttk.Frame(self.master)
        self.notebook.add(self.frame3, text='Uploader')
        
        #dump generator tab (1)
        self.labelframe11 = LabelFrame(self.frame1, text="Single download")
        self.labelframe11.grid(row=0, column=0)
        self.labelframe12 = LabelFrame(self.frame1, text="Batch download")
        self.labelframe12.grid(row=1, column=0)
        #single download labelframe
        self.label11 = Label(self.labelframe11, text="Wiki URL:")
        self.label11.grid(row=0, column=0)
        self.entry11 = Entry(self.labelframe11, width=40)
        self.entry11.grid(row=0, column=1)
        self.entry11.bind('<Return>', (lambda event: self.checkURL()))
        self.optionmenu11var = StringVar(self.labelframe11)
        self.optionmenu11var.set("api.php")
        self.optionmenu11 = OptionMenu(self.labelframe11, self.optionmenu11var, self.optionmenu11var.get(), "index.php")
        self.optionmenu11.grid(row=0, column=2)
        self.button11 = Button(self.labelframe11, text="Check", command=lambda: thread.start_new_thread(self.checkURL, ()), width=5)
        self.button11.grid(row=0, column=3)
        #batch download labelframe
        self.label12 = Label(self.labelframe12, text="Wiki URLs:")
        self.label12.grid(row=0, column=0)
        self.text11 = Text(self.labelframe12, width=70, height=20)
        self.text11.grid(row=0, column=1)
        
        #downloader tab (2)
        self.label25var = StringVar(self.frame2)
        self.label25var.set("Available dumps: 0 (0.0 MB)")
        self.label25 = Label(self.frame2, textvariable=self.label25var, width=27, anchor=W)
        self.label25.grid(row=0, column=0, columnspan=2)
        self.label26var = StringVar(self.frame2)
        self.label26var.set("Downloaded: 0 (0.0 MB)")
        self.label26 = Label(self.frame2, textvariable=self.label26var, background='lightgreen', width=27, anchor=W)
        self.label26.grid(row=0, column=2, columnspan=2)
        self.label27var = StringVar(self.frame2)
        self.label27var.set("Not downloaded: 0 (0.0 MB)")
        self.label27 = Label(self.frame2, textvariable=self.label27var, background='white', width=27, anchor=W)
        self.label27.grid(row=0, column=4, columnspan=2)
        
        self.label21 = Label(self.frame2, text="Filter by wikifarm:", width=15, anchor=W)
        self.label21.grid(row=1, column=0)
        self.optionmenu21var = StringVar(self.frame2)
        self.optionmenu21var.set("all")
        self.optionmenu21 = OptionMenu(self.frame2, self.optionmenu21var, self.optionmenu21var.get(), "Gentoo Wiki", "OpenSuSE", "Referata", "ShoutWiki", "Unknown", "Wikanda", "WikiFur", "Wikimedia", "WikiTravel", "Wikkii")
        self.optionmenu21.grid(row=1, column=1)
        
        self.label22 = Label(self.frame2, text="Filter by size:", width=15, anchor=W)
        self.label22.grid(row=1, column=2)
        self.optionmenu22var = StringVar(self.frame2)
        self.optionmenu22var.set("all")
        self.optionmenu22 = OptionMenu(self.frame2, self.optionmenu22var, self.optionmenu22var.get(), "KB", "MB", "GB", "TB")
        self.optionmenu22.grid(row=1, column=3)
        
        self.label23 = Label(self.frame2, text="Filter by date:", width=15, anchor=W)
        self.label23.grid(row=1, column=4)
        self.optionmenu23var = StringVar(self.frame2)
        self.optionmenu23var.set("all")
        self.optionmenu23 = OptionMenu(self.frame2, self.optionmenu23var, self.optionmenu23var.get(), "2011", "2012")
        self.optionmenu23.grid(row=1, column=5)
        
        self.label24 = Label(self.frame2, text="Filter by mirror:")
        self.label24.grid(row=1, column=6)
        self.optionmenu24var = StringVar(self.frame2)
        self.optionmenu24var.set("all")
        self.optionmenu24 = OptionMenu(self.frame2, self.optionmenu24var, self.optionmenu24var.get(), "Google Code", "Internet Archive", "ScottDB")
        self.optionmenu24.grid(row=1, column=7)
        
        self.button23 = Button(self.frame2, text="Filter!", command=self.filterAvailableDumps, width=7)
        self.button23.grid(row=1, column=8)
        
        self.treescrollbar = Scrollbar(self.frame2)
        self.treescrollbar.grid(row=2, column=9, sticky=W+E+N+S)
        columns = ('dump', 'wikifarm', 'size', 'date', 'mirror', 'status')
        self.tree = ttk.Treeview(self.frame2, height=20, columns=columns, show='headings', yscrollcommand=self.treescrollbar.set)
        self.treescrollbar.config(command=self.tree.yview)
        self.tree.column('dump', width=495, minwidth=200, anchor='center')
        self.tree.heading('dump', text='Dump')
        self.tree.column('wikifarm', width=100, minwidth=100, anchor='center')
        self.tree.heading('wikifarm', text='Wikifarm')
        self.tree.column('size', width=100, minwidth=100, anchor='center')
        self.tree.heading('size', text='Size')
        self.tree.column('date', width=100, minwidth=100, anchor='center')
        self.tree.heading('date', text='Date')
        self.tree.column('mirror', width=120, minwidth=120, anchor='center')
        self.tree.heading('mirror', text='Mirror')
        self.tree.column('status', width=120, minwidth=120, anchor='center')
        self.tree.heading('status', text='Status')
        self.tree.grid(row=2, column=0, columnspan=9, sticky=W+E+N+S)
        [self.tree.heading(column, text=column, command=lambda: self.treeSortColumn(column=column, reverse=False)) for column in columns]        
        #self.tree.bind("<Double-1>", (lambda: thread.start_new_thread(self.downloadDump, ())))
        self.tree.tag_configure('downloaded', background='lightgreen')
        self.tree.tag_configure('nodownloaded', background='white')
        self.button21 = Button(self.frame2, text="Load available dumps", command=lambda: thread.start_new_thread(self.loadAvailableDumps, ()), width=15)
        self.button21.grid(row=3, column=0)
        self.button23 = Button(self.frame2, text="Download selection", command=lambda: thread.start_new_thread(self.downloadDump, ()), width=15)
        self.button23.grid(row=3, column=4)
        self.button22 = Button(self.frame2, text="Clear list", command=self.deleteAvailableDumps, width=10)
        self.button22.grid(row=3, column=8, columnspan=2)
        
        #uploader tab (3)
        self.label31 = Label(self.frame3, text="todo...")
        self.label31.grid(row=0, column=0)
        #end tabs
        
        #statusbar
        self.status = Label(self.master, text="Welcome to WikiTeam tools. What wiki do you want to preserve today?", bd=1, background='grey', justify=LEFT, relief=SUNKEN)
        self.status.grid(row=4, column=0, columnspan=10, sticky=W+E)
        
        #begin menu
        menu = Menu(self.master)
        master.config(menu=menu)

        #file menu
        filemenu = Menu(menu)
        menu.add_cascade(label="File", menu=filemenu)
        filemenu.add_command(label="Preferences", command=self.callback)
        filemenu.add_separator()
        filemenu.add_command(label="Exit", command=askclose)
        
        #help menu
        helpmenu = Menu(menu)
        menu.add_cascade(label="Help", menu=helpmenu)
        helpmenu.add_command(label="About", command=self.callback)
        helpmenu.add_command(label="Help index", command=self.callback)
        helpmenu.add_command(label="WikiTeam homepage", command=lambda: webbrowser.open_new_tab(HOMEPAGE))
        #end menu
    
    def blocked(self):
        tkMessageBox.showerror("Error", "There is a task in progress. Please, wait.")
    
    def checkURL(self):
        if re.search(ur"(?im)^https?://[^/]+\.[^/]+/", self.entry11.get()): #well-constructed URL?, one dot at least, aaaaa.com, but bb.aaaaa.com is allowed too
            if self.optionmenu11var.get() == 'api.php':
                self.msg('Please wait... Checking api.php...')
                if dumpgenerator.checkAPI(self.entry11.get()):
                    self.entry11.config(background='lightgreen')
                    self.msg('api.php is correct!', level='ok')
                else:
                    self.entry11.config(background='red')
                    self.msg('api.php is incorrect!', level='error')
            elif self.optionmenu11var.get() == 'index.php':
                self.msg('Please wait... Checking index.php...')
                if dumpgenerator.checkIndexphp(self.entry11.get()):
                    self.entry11.config(background='lightgreen')
                    self.msg('index.php is OK!', level='ok')
                else:
                    self.entry11.config(background='red')
                    self.msg('index.php is incorrect!', level='error')
        else:
            tkMessageBox.showerror("Error", "You have to write a correct api.php or index.php URL.")
    
    def sumSizes(self, sizes):
        total = 0
        for size in sizes:
            if size.endswith('KB'):
                total += float(size.split(' ')[0])
            elif size.endswith('MB'):
                total += float(size.split(' ')[0])*1024
            elif size.endswith('GB'):
                total += float(size.split(' ')[0])*1024*1024
            elif size.endswith('TB'):
                total += float(size.split(' ')[0])*1024*1024*1024
            elif not size or size.lower() == 'unknown':
                pass
            else:
                total += size
        return total/1024 #MB
    
    def run(self):
        for i in range(10):
            time.sleep(0.1)
            self.value += 10
        
        """
        #get parameters selected
        params = ['--api=http://www.archiveteam.org/api.php', '--xml']

        #launch dump
        dumpgenerator.main(params=params)

        #check dump
        """
    
    def msg(self, msg='', level=''):
        levels = { 'ok': 'lightgreen', 'warning': 'yellow', 'error': 'red' }
        if levels.has_key(level.lower()):
            print '%s: %s' % (level.upper(), msg)
            self.status.config(text='%s: %s' % (level.upper(), msg), background=levels[level.lower()])
        else:
            print msg
            self.status.config(text=msg, background='grey')
    
    def treeSortColumn(self, column, reverse=False):
        l = [(self.tree.set(i, column), i) for i in self.tree.get_children('')]
        l.sort(reverse=reverse)
        for index, (val, i) in enumerate(l):
            self.tree.move(i, '', index)
        self.tree.heading(column, command=lambda: self.treeSortColumn(column=column, reverse=not reverse))
    
    def downloadProgress(self, block_count, block_size, total_size):
        try:
            total_mb = total_size/1024/1024.0
            downloaded = block_count *(block_size/1024/1024.0)
            percent = downloaded/(total_mb/100.0)
            if not random.randint(0,10):
                msg = "%.1f MB of %.1f MB downloaded (%.1f%%)" % (downloaded, total_mb, percent <= 100 and percent or 100)
                self.msg(msg, level='ok')
            #sys.stdout.write("%.1f MB of %.1f MB downloaded (%.2f%%)" %(downloaded, total_mb, percent))
            #sys.stdout.flush()
        except:
            pass
    
    def downloadDump(self, event=None):
        if self.block:
            self.blocked()
            return
        else:
            self.block = True
        items = self.tree.selection()
        if items:
            if not os.path.exists(self.downloadpath):
                os.makedirs(self.downloadpath)
            c = 0
            d = 0
            for item in items:
                filepath = self.downloadpath and self.downloadpath + '/' + self.dumps[int(item)][0] or self.dumps[int(item)][0]
                if os.path.exists(filepath):
                    self.msg('That dump was downloaded before', level='ok')
                    d += 1
                else:
                    self.msg("[%d of %d] Downloading %s from %s" % (c+1, len(items), self.tree.item(item,"text"), self.dumps[int(item)][5]))
                    f = urllib.urlretrieve(self.dumps[int(item)][5], filepath, reporthook=self.downloadProgress)
                    msg='%s size is %s bytes large. Download successful!' % (self.dumps[int(item)][0], os.path.getsize(filepath))
                    self.msg(msg=msg, level='ok')
                    c += 1
                self.dumps[int(item)] = self.dumps[int(item)][:6] + ['True']
            if c + d == len(items):
                self.msg('Downloaded %d of %d%s.' % (c, len(items), d and ' (and %d were previously downloaded)' % (d) or ''), level='ok')
            else:
                self.msg('Problems in %d dumps. Downloaded %d of %d (and %d were previously downloaded).' % (len(items)-(c+d), c, len(items), d), level='error')
        else:
            tkMessageBox.showerror("Error", "You have to select some dumps to download.")
        self.clearAvailableDumps()
        self.showAvailableDumps()
        self.filterAvailableDumps()
        self.block = False
    
    def deleteAvailableDumps(self):
        #really delete dump list and clear tree
        self.clearAvailableDumps()
        self.dumps = [] #reset list
    
    def clearAvailableDumps(self):
        #clear tree
        for i in range(len(self.dumps)):
            self.tree.delete(str(i))
    
    def showAvailableDumps(self):
        c = 0
        for filename, wikifarm, size, date, mirror, url, downloaded in self.dumps:
            self.tree.insert('', 'end', str(c), text=filename, values=(filename, wikifarm, size, date, mirror, downloaded and 'Downloaded' or 'Not downloaded'), tags=(downloaded and 'downloaded' or 'nodownloaded',))
            c += 1
        
    def filterAvailableDumps(self):
        self.clearAvailableDumps()
        self.showAvailableDumps()
        sizes = []
        downloadedsizes = []
        nodownloadedsizes = []
        for i in range(len(self.dumps)):
            if (self.optionmenu21var.get() == 'all' and self.optionmenu22var.get() == 'all' and self.optionmenu23var.get() == 'all' and self.optionmenu24var.get() == 'all'):
                sizes.append(self.dumps[i][2])
                if self.dumps[i][6]:
                    downloadedsizes.append(self.dumps[i][2])
                else:
                    nodownloadedsizes.append(self.dumps[i][2])
            elif (self.optionmenu21var.get() != 'all' and not self.optionmenu21var.get() == self.dumps[i][1]) or \
                (self.optionmenu22var.get() != 'all' and not self.optionmenu22var.get() in self.dumps[i][2]) or \
                (self.optionmenu23var.get() != 'all' and not self.optionmenu23var.get() in self.dumps[i][3]) or \
                (self.optionmenu24var.get() != 'all' and not self.optionmenu24var.get() in self.dumps[i][4]):
                self.tree.detach(str(i)) #hide this item
                sizes.append(self.dumps[i][2])
                if self.dumps[i][6]:
                    downloadedsizes.append(self.dumps[i][2])
                else:
                    nodownloadedsizes.append(self.dumps[i][2])
        self.label25var.set("Available dumps: %d (%.1f MB)" % (len(sizes), self.sumSizes(sizes)))
        self.label26var.set("Downloaded: %d (%.1f MB)" % (len(downloadedsizes), self.sumSizes(downloadedsizes)))
        self.label27var.set("Not downloaded: %d (%.1f MB)" % (len(nodownloadedsizes), self.sumSizes(nodownloadedsizes)))
    
    def isDumpDownloaded(self, filename):
        #improve, size check or md5sum?
        if filename:
            filepath = self.downloadpath and self.downloadpath + '/' + filename or filename
            if os.path.exists(filepath):
                return True
        
        """estsize = os.path.getsize(filepath)
                c = 0
                while int(estsize) >= 1024:
                    estsize = estsize/1024.0
                    c += 1
                estsize = '%.1f %s' % (estsize, ['', 'KB', 'MB', 'GB', 'TB'][c])"""
        
        return False
    
    def loadAvailableDumps(self):
        if self.block:
            self.blocked()
            return
        else:
            self.block = True
        if self.dumps:
            self.deleteAvailableDumps()
        iaregexp = ur'/download/[^/]+/(?P<filename>[^>]+\.7z)">\s*(?P<size>[\d\.]+ (?:KB|MB|GB|TB))\s*</a>'
        self.urls = [
            ['Google Code', 'https://code.google.com/p/wikiteam/downloads/list?num=5000&start=0', ur'(?im)detail\?name=(?P<filename>[^&]+\.7z)&amp;can=2&amp;q=" style="white-space:nowrap">\s*(?P<size>[\d\.]+ (?:KB|MB|GB|TB))\s*</a></td>'],
            ['Internet Archive', 'http://www.archive.org/details/referata.com-20111204', iaregexp],
            ['Internet Archive', 'http://www.archive.org/details/WikiTeamMirror', iaregexp],
            ['ScottDB', 'http://mirrors.sdboyd56.com/WikiTeam/', ur'<a href="(?P<filename>[^>]+\.7z)">(?P<size>[\d\.]+ (?:KB|MB|GB|TB))</a>'],
            ['Wikimedia', 'http://dumps.wikimedia.org/backup-index.html', ur'(?P<size>)<a href="(?P<filename>[^>]+)">[^>]+</a>: <span class=\'done\'>Dump complete</span></li>']
        ]
        wikifarms_r = re.compile(ur"(%s)" % ('|'.join(wikifarms.keys())))
        c = 0
        for mirror, url, regexp in self.urls:
            print 'Loading data from', mirror, url
            self.msg(msg='Please wait... Loading data from %s %s' % (mirror, url))
            f = urllib.urlopen(url)
            m = re.compile(regexp).finditer(f.read())
            for i in m:
                filename = i.group('filename')
                if mirror == 'Wikimedia':
                    filename = '%s-pages-meta-history.xml.7z' % (re.sub('/', '-', filename))
                wikifarm = 'Unknown'
                if re.search(wikifarms_r, filename):
                    wikifarm = re.findall(wikifarms_r, filename)[0]
                wikifarm = wikifarms[wikifarm]
                size = i.group('size')
                if not size:
                    size = 'Unknown'
                date = 'Unknown'
                if re.search(ur"\-(\d{8})[\.-]", filename):
                    date = re.findall(ur"\-(\d{4})(\d{2})(\d{2})[\.-]", filename)[0]
                    date = '%s-%s-%s' % (date[0], date[1], date[2])
                elif re.search(ur"\-(\d{4}\-\d{2}\-\d{2})[\.-]", filename):
                    date = re.findall(ur"\-(\d{4}\-\d{2}\-\d{2})[\.-]", filename)[0]
                downloadurl = ''
                if mirror == 'Google Code':
                    downloadurl = 'https://wikiteam.googlecode.com/files/' + filename
                elif mirror == 'Internet Archive':
                    downloadurl = re.sub(ur'/details/', ur'/download/', url) + '/' + filename
                elif mirror == 'ScottDB':
                    downloadurl = url + '/' + filename
                elif mirror == 'Wikimedia':
                    downloadurl = 'http://dumps.wikimedia.org/' + filename.split('-')[0] + '/' + re.sub('-', '', date) + '/' + filename
                downloaded = self.isDumpDownloaded(filename)
                self.dumps.append([filename, wikifarm, size, date, mirror, downloadurl, downloaded])
        self.dumps.sort()
        self.showAvailableDumps()
        self.filterAvailableDumps()
        self.msg(msg='Loaded %d available dumps!' % (len(self.dumps)), level='ok')
        self.block = False
    
    def callback(self):
        self.msg("Feature not implemented for the moment. Contributions are welcome.", level='warning')
    
def askclose():
    if tkMessageBox.askokcancel("Quit", "Do you really wish to exit?"):
        root.destroy()

if __name__ == "__main__":
    root = Tk()
    width = 1050
    height = 560
    # calculate position x, y
    x = (root.winfo_screenwidth()/2) - (width/2) 
    y = (root.winfo_screenheight()/2) - (height/2)
    root.geometry('%dx%d+%d+%d' % (width, height, x, y))
    root.title('%s (version %s)' % (NAME, VERSION))
    root.protocol("WM_DELETE_WINDOW", askclose)
    #logo
    #imagelogo = PhotoImage(file = 'logo.gif')
    #labellogo = Label(root, image=imagelogo)
    #labellogo.grid(row=0, column=0, rowspan=3, sticky=W)
    app = App(root)
    root.mainloop()