From d5b17717c0ecab1f07498ae9e2ae17b06dc95bb1 Mon Sep 17 00:00:00 2001 From: Michael McGrath Date: Wed, 30 Apr 2008 14:14:27 -0500 Subject: [PATCH 1/4] Added more changes to the moin2mw scritp for ticket #31 --- scripts/moin2mw/moin2mw.py | 82 +++++++++++++++++++++++++++++++++++--- 1 file changed, 77 insertions(+), 5 deletions(-) diff --git a/scripts/moin2mw/moin2mw.py b/scripts/moin2mw/moin2mw.py index 26d3d9d..4a5e4e5 100755 --- a/scripts/moin2mw/moin2mw.py +++ b/scripts/moin2mw/moin2mw.py @@ -55,10 +55,10 @@ def _table_xlat(data): return u''.join(result) def _escape(line): + return (line, {}) # line = line.replace(u">", u">") # line = line.replace(u"<", u"<") # line = re.sub(ur'&(?![a-z]+;)', u"&", line) - return (line, {}) def _fix_comments(line): if line.startswith(u"##"): @@ -66,6 +66,11 @@ def _fix_comments(line): return (line, {}) +def _fix_redirect(line): + if line.startswith(u"#redirect"): + line = u"#REDIRECT %s" % line.split(" ")[2:] + return(line, {}) + def _find_meta(line): try: if line.startswith(u"#"): @@ -81,6 +86,18 @@ def _studlycaps(line): # ur'[[\1]]', line) return (line, {}) +def _fix_anchors(line): + + if line.find('[[Anchor') != -1 or line.find('[[ Anchor') != -1: + line = line.replace('[[', '{{') + line = line.replace(')]]', '}}') + line = line.replace(') ]]', '}}') + line = line.replace('(', '|') +# if line.find('<\>', '}}', line, 1)[0] + return (line, {}) + def _fix_bullets(line): if re.match(ur'^\s+\*', line): while line[0].isspace(): @@ -96,14 +113,44 @@ def _fix_numlists(line): return (line, {}) +def _fix_admonition(line): +# while line.find(ur'[[Admonition') != -1: +# line = re.subn(ur'\[\[Admonition', '<>', line, 1)[0] + return (line, {}) + +def _fix_get_val(line): + split_line = line.split(']]') + e = [] + for s in split_line: + if s.find('[[GetVal') != -1: + s = s.replace('[[GetVal', '{{Template:',1) + s = s.replace(',', '/', 1) + s = s.replace('(', '', 1) + s = s.replace(')', '', 1) + s = s + '}}' + e.append(s) + line = ' '.join(e) + + return (line, {}) + def _fix_pre(line): - if line.find('{{{') and line.find('}}}'): + if line.find('{{{') != -1 and line.find('}}}') != -1: line = re.sub(r'\{\{\{', "", line) line = re.sub(r'\}\}\}', "", line) else: line = re.sub(r'\{\{\{', "
", line)
         line = re.sub(r'\}\}\}', "
", line) + return (line, {}) +#def _fix_big_links(line): +# line = re.sub(ur'\[:([^:]+):([^]]+)]', ur'[\1|\2]', line) +# return (line, {}) + +def _fix_code_blocks(line): + while line.find('`') != -1: + line = re.subn(ur'`', '', line, 1)[0] + line = re.subn(ur'`', '', line, 1)[0] return (line, {}) def _unspace_text(line): @@ -119,8 +166,33 @@ def _kill_link_prefixes(line): line = re.sub(ur'[A-Za-z]+\:\[\[', u"[[", line) return (line, {}) +def _fix_line_breaks(line): + line = line.replace('[[BR]]', '
') + return (line, {}) + +def _fix_headers(line): + ''' This is Fedora specific ''' + line = line.replace('

', '==') + line = line.replace('

', '==') + line = line.replace('

', '==') + line = line.replace('

', '==') + return (line, {}) + def _fix_links(line): - line = re.sub(ur'\[\:(.*)\:(.*)\]', ur"[[\1 |\2]]", line) + split_line = line.split(']') +#while f.find('[:') != -1: + e = [] + for s in split_line: + if s.find('[:') != -1: + s = s.replace('[:', '[[',1) + s = s.replace(':', '| ', 1) + s = s.replace(']', ']]', 1) + s = s + ']]' + elif s.find('[') != -1: + s = s + ']' + e.append(s) + line = ' '.join(e) +# line = re.sub(ur'\[\:(.*)\:(.*)\]', ur"[[\1 |\2]]", line) # line = re.sub(r'\[\[', "[[ ", line) # line = re.sub(r'\]\]', " ]]", line) return (line, {}) @@ -131,8 +203,8 @@ def _remove_toc(line): return (line, {}) -chain = [ _remove_toc, _fix_links, _escape, _fix_comments, _find_meta, _studlycaps, _fix_bullets, - _fix_numlists, _fix_pre, _unspace_text, _kill_link_prefixes ] +chain = [ _fix_line_breaks, _fix_headers, _fix_anchors, _remove_toc, _fix_get_val, _fix_links, _escape, _fix_redirect, _fix_comments, _find_meta, _studlycaps, _fix_bullets, + _fix_numlists, _unspace_text, _kill_link_prefixes, _fix_code_blocks, _fix_pre, _fix_admonition ] class MoinWiki(object): def __init__(self, wiki_path): From 01391cc7108c21dc5af2820cce47f248bc6ed81c Mon Sep 17 00:00:00 2001 From: Mike McGrath Date: Thu, 1 May 2008 13:44:44 -0500 Subject: [PATCH 2/4] Added snap check status script --- scripts/checkSnapStatus/checkSnapStatus.sh | 34 ++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100755 scripts/checkSnapStatus/checkSnapStatus.sh diff --git a/scripts/checkSnapStatus/checkSnapStatus.sh b/scripts/checkSnapStatus/checkSnapStatus.sh new file mode 100755 index 0000000..3e64062 --- /dev/null +++ b/scripts/checkSnapStatus/checkSnapStatus.sh @@ -0,0 +1,34 @@ +#!/bin/bash +NETAPPMIB='/home/fedora/mmcgrath/netapp.mib' +COMMUNITY='public' +HOST='ntap-fedora1' + +function get { + snmpget -v 1 -Ov -m $NETAPPMIB -c $COMMUNITY $HOST $1 | awk -F": " '{ print $2 }' +} + +echo "PHX" +echo -n " Status: " +get .1.3.6.1.4.1.789.1.9.20.1.4.1 +echo -n " Lag: " +get .1.3.6.1.4.1.789.1.9.20.1.6.1 +echo -n " Mirror Timestamp: " +get .1.3.6.1.4.1.789.1.9.20.1.14.1 +echo -n " Last Transfer Size (MB): " +get .1.3.6.1.4.1.789.1.9.20.1.17.1 +echo -n " Last Transfer Time (seconds): " +get .1.3.6.1.4.1.789.1.9.20.1.18.1 + +echo "TPA" +echo -n " Status: " +get .1.3.6.1.4.1.789.1.9.20.1.4.2 +echo -n " Lag: " +get .1.3.6.1.4.1.789.1.9.20.1.6.2 +echo -n " Mirror Timestamp: " +get .1.3.6.1.4.1.789.1.9.20.1.14.2 +echo -n " Last Transfer Size (MB): " +get .1.3.6.1.4.1.789.1.9.20.1.17.2 +echo -n " Last Transfer Time (seconds): " +get .1.3.6.1.4.1.789.1.9.20.1.18.2 + + From b3a480be3e5fc8ee8e71d3f35de784c5ebbcf6d5 Mon Sep 17 00:00:00 2001 From: Mike McGrath Date: Mon, 5 May 2008 13:41:29 -0500 Subject: [PATCH 3/4] further changes in mediawiki and initial upload script is ready --- scripts/moin2mw/moin2mw.py | 66 +++++++++++++++++++++++++++++++----- scripts/moin2mw/mw-upload.py | 47 +++++++++++++++++++++++++ scripts/moin2mw/upAttach.sh | 8 +++++ 3 files changed, 112 insertions(+), 9 deletions(-) create mode 100755 scripts/moin2mw/mw-upload.py create mode 100644 scripts/moin2mw/upAttach.sh diff --git a/scripts/moin2mw/moin2mw.py b/scripts/moin2mw/moin2mw.py index 4a5e4e5..86ab2b7 100755 --- a/scripts/moin2mw/moin2mw.py +++ b/scripts/moin2mw/moin2mw.py @@ -23,12 +23,13 @@ def _table_xlat(data): result = [] #sys.stderr.write("Data: %s" % data) for line in data.splitlines(True): - if line.startswith(u"||"): + if line.strip(':').strip().startswith(u"||"): if not in_table: - if line.startswith(u"||", line) @@ -170,6 +179,15 @@ def _fix_line_breaks(line): line = line.replace('[[BR]]', '
') return (line, {}) +def _fix_categories(line): + if line.startswith('Category'): + line = line.replace('Category', '[[Category:') + line = line.strip() + ']]' + + #line = line.replace('\n', ']]') + + return (line, {}) + def _fix_headers(line): ''' This is Fedora specific ''' line = line.replace('

', '==') @@ -179,6 +197,8 @@ def _fix_headers(line): return (line, {}) def _fix_links(line): + if line.find('Category:') != -1: + return (line, {}) split_line = line.split(']') #while f.find('[:') != -1: e = [] @@ -198,13 +218,36 @@ def _fix_links(line): return (line, {}) def _remove_toc(line): + return (line, {}) if not line.find('TableOfContents') == -1: line = re.sub(r'\[\[.*TableOfContents.*\]\]', '', line) return (line, {}) +def _fix_attachments(line, page_name): + result = [] + if line.find('attachment:') != -1: + dest = page_name.replace('(2f)', '_') + '_' + skipFirst=1 + for l in line.split('attachment:'): + if skipFirst==1: + result.append(l) + skipFirst=0 + continue + l = "[[Image:%s%s" % (dest, l) + l = re.subn(ur'([A-Za-z0-9:_\.\-]*)([A-Za-z0-9])', ur'\1\2]]', l, 1)[0] + result.append(l) + line = ''.join(result) + # crazy hack, fix triples (they happen from linked moin images) + line = line.replace('[[[', '[[') + line = line.replace(']]]', ']]') + return (line) -chain = [ _fix_line_breaks, _fix_headers, _fix_anchors, _remove_toc, _fix_get_val, _fix_links, _escape, _fix_redirect, _fix_comments, _find_meta, _studlycaps, _fix_bullets, - _fix_numlists, _unspace_text, _kill_link_prefixes, _fix_code_blocks, _fix_pre, _fix_admonition ] + +chain = [ _remove_toc, _fix_line_breaks, _fix_categories, _fix_headers, _fix_anchors, + _fix_include, _fix_get_val, _fix_links, _escape, + _fix_redirect, _fix_comments, _find_meta, _studlycaps, _fix_bullets, + _fix_numlists, _unspace_text, _kill_link_prefixes, _fix_code_blocks, + _fix_pre, _fix_admonition ] class MoinWiki(object): def __init__(self, wiki_path): @@ -246,13 +289,17 @@ class MoinWiki(object): return new_page_name - def _chain_translate_file(self, f): + def _chain_translate_file(self, f, page_name): result = [] resultmeta = {} + if page_name.find('MoinEditorBackup') != -1: + return (result, resultmeta) for line in f: for chaincall in chain: (line, meta) = chaincall(line) resultmeta.update(meta) + # fix_attachments is fedora specific and requites pagename + line = _fix_attachments(line, page_name) result.append(line) result = _table_xlat(u''.join(result)) @@ -299,7 +346,8 @@ class MoinWiki(object): text_file = codecs.open(u"%s/revisions/%s" % (page_path, revnum), 'r', 'utf-8') (results[u"text"], results[u"meta"]) = \ - self._chain_translate_file(text_file) + self._chain_translate_file(text_file, orig_page_name) + #sys.stderr.write("page_path: %s\n" % orig_page_name) text_file.close() return results diff --git a/scripts/moin2mw/mw-upload.py b/scripts/moin2mw/mw-upload.py new file mode 100755 index 0000000..9d8e015 --- /dev/null +++ b/scripts/moin2mw/mw-upload.py @@ -0,0 +1,47 @@ +#!/usr/bin/python -tt + +# written by seth vidal +# Altered by Mike McGrath +import mechanize +import sys + +try: + sys.argv[2] +except IndexError: + print "Please specify [source] [dest name]" + sys.exit() +try: + sys.argv[3] +except IndexError: + pass +else: + print "Please specify [source] [dest name]" + sys.exit() + +try: + f = open(sys.argv[1]) +except IOError: + print "Could not open %s" % sys.argv[1] + sys.exit() + +b = mechanize.Browser(factory=mechanize.DefaultFactory(i_want_broken_xhtml_support=True)) +b.set_handle_robots(False) +b.open("https://publictest1.fedoraproject.org/wiki/Special:Userlogin") +b.select_form(nr=1) +b["wpName"] = "admin" +b["wpPassword"] = "adminadmin" +b.submit() +r = b.response() + +b.open("https://publictest1.fedoraproject.org/wiki/Special:Upload") +b.select_form(nr=1) +b["wpDestFile"] = sys.argv[2] +b['wpUploadDescription'] = 'Migrated from previous wiki' +b.form.add_file(open(sys.argv[1]), filename=sys.argv[1]) +b.submit() +r = b.response() +results='\n'.join(r.readlines()) +if results.find('Success') != -1 or results.find('Migrated from previous wiki') != -1: + print "%s - Success" % sys.argv[1] +else: + print "%s - Failure" % sys.argv[1] diff --git a/scripts/moin2mw/upAttach.sh b/scripts/moin2mw/upAttach.sh new file mode 100644 index 0000000..ee4fa03 --- /dev/null +++ b/scripts/moin2mw/upAttach.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +for f in `find -type f | grep attachment` +do + dest=`echo $f | sed -e 's,/attachments,,' -e 's,./,,' -e 's/(2f)/_/' -e 's,/,_,g'` + src=$f + echo "/root/mw-upload.py '$src' '$dest'" +done From 17bf6b34059be94685bea44649307aed77d0e840 Mon Sep 17 00:00:00 2001 From: Michael McGrath Date: Mon, 5 May 2008 17:38:13 -0500 Subject: [PATCH 4/4] Much better support for tables now and notice classes should be working with proper templates --- scripts/moin2mw/moin2mw.py | 52 ++++++++++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/scripts/moin2mw/moin2mw.py b/scripts/moin2mw/moin2mw.py index 86ab2b7..ae9a744 100755 --- a/scripts/moin2mw/moin2mw.py +++ b/scripts/moin2mw/moin2mw.py @@ -24,31 +24,53 @@ def _table_xlat(data): #sys.stderr.write("Data: %s" % data) for line in data.splitlines(True): if line.strip(':').strip().startswith(u"||"): + # Gut the html stuff until we figure out what to do with it + #line = re.sub(r'<([a-zA-Z\/][^>])*>', '\1|', line) if not in_table: if line.strip().startswith(u"||]*)>.*', r'\1', line.split('>')[0] + '>') + tableclass = re.sub(r'.*<([a-zA-Z\/][^>]*)>(.*)', r'\1 | \2', line).replace('tableclass="', 'Template:').replace(' ', '/', 1).replace('"', '', 1) + result.append(u"{{ %s" % tableclass) has_class = True elif line.strip().startswith(u"||]*)>.*', r'\1', line.split('>')[0] + '>') + tablestyle = tablestyle.replace('tablestyle', 'style') + result.append(u"{| %s" % tablestyle) else: - result.append(u"{| border=\"1\"\n") + result.append(u"{| border=\"1\"") in_table = True newline = line[1:] while newline[-1] in (u"|", u" "): newline = newline[:-1] - newline = re.sub('\', '', newline) - newline = re.sub('\', '', newline) + #newline = re.sub('\', '', newline) + #newline = re.sub('\', '', newline) newline = newline.rstrip().rstrip('||').rstrip() - result.append(newline) - result.append(u"\n|-\n") + #newline = re.sub(r'.*<([a-zA-Z\/][^>]*)>.*', r'\1 |', newline) + #newline = re.sub(r'<([a-zA-Z\/][^>]*)>', r'\1 |', newline) + newline = re.sub(r']*>', r'', newline) + # Ugh nasty + if newline.find('rowstyle') != -1: + newline = re.sub(r'<([a-zA-Z\/][^>]*)>', r'\1 \n|', newline) + newline = newline.replace('|rowstyle', 'style') + newline = newline.replace('| rowstyle', 'style') + result.append(u"\n|- %s" % newline) + else: + newline = newline.replace('', '" |') + newline = newline.replace('" >', '" |') + newline = newline.replace("'>", "' |") + newline = newline.replace("' >", "' |") + if not has_class: + result.append(u"\n|-") + result.append("\n" + newline) else: if in_table: if has_class: - result.append(u"}}\n") + result.append(u"\n}}\n") else: - result.append(u"|}\n") + result.append(u"\n|}\n") in_table = False has_class=False result.append(line) @@ -203,13 +225,18 @@ def _fix_links(line): #while f.find('[:') != -1: e = [] for s in split_line: +# sys.stderr.write("s before: %s\n" % s) if s.find('[:') != -1: s = s.replace('[:', '[[',1) - s = s.replace(':', '| ', 1) + tmp = s.split('[[') + #sys.stderr.write("0: " + tmp[0]) + #sys.stderr.write("1: " + tmp[1]) + s = tmp[0] + '[[' + tmp[1].replace(':', '| ', 1) s = s.replace(']', ']]', 1) s = s + ']]' elif s.find('[') != -1: s = s + ']' + #sys.stderr.write("s after: %s\n" % s) e.append(s) line = ' '.join(e) # line = re.sub(ur'\[\:(.*)\:(.*)\]', ur"[[\1 |\2]]", line) @@ -218,9 +245,8 @@ def _fix_links(line): return (line, {}) def _remove_toc(line): - return (line, {}) - if not line.find('TableOfContents') == -1: - line = re.sub(r'\[\[.*TableOfContents.*\]\]', '', line) + line = line.replace('[[TableOfContents]]', '') + line = line.replace('[[ TableOfContents ]]', '') return (line, {}) def _fix_attachments(line, page_name):