Subversion Repositories basico

Compare Revisions

Ignore whitespace Rev 362 → Rev 363

/branches/BR-0.4/basico/services/srv_annot.py
File deleted
/branches/BR-0.4/basico/services/srv_db.py
File deleted
/branches/BR-0.4/basico/services/srv_cols.py
File deleted
/branches/BR-0.4/basico/basico.py
24,8 → 24,8
from basico.services.srv_notify import Notification
from basico.services.srv_db import Database
from basico.services.srv_driver import SeleniumDriver
from basico.services.srv_cols import Collections
from basico.services.srv_annot import Annotation
from basico.services.srv_collections import Collections
from basico.services.srv_annotations import Annotation
from basico.services.srv_attachment import Attachment
from basico.services.srv_notify import Notification
 
/branches/BR-0.4/basico/services/srv_annotations.py
0,0 → 1,237
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
# File: srv_annot.py
# Author: Tomás Vírseda
# License: GPL v3
# Description: Annotations service
"""
 
import os
import json
import uuid
import glob
from os.path import sep as SEP
 
from basico.core.mod_env import FILE, LPATH
from basico.core.mod_srv import Service
 
 
class Annotation(Service):
def initialize(self):
'''
Setup Annotation Service
'''
self.get_services()
self.__fix_annotations()
 
 
def get_services(self):
self.srvutl = self.get_service('Utils')
 
 
def __fix_annotations(self):
"""
In Basico 0.4, new field 'created' is introduced.
For annotations created before, created timestamp = updated timestamp.
"""
 
for filename in self.get_all():
metadata = self.get_metadata_from_file(filename)
try:
ts = metadata['Created']
except Exception as error:
# Fix annotation metadata: add 'Created' field
metadata['Created'] = metadata['Timestamp']
with open(filename, 'w') as fa:
json.dump(metadata, fa)
self.log.debug("Fixed annotation with AID: %s", metadata['AID'])
 
def gen_aid(self, sid):
'''
Generate new annotation id
'''
return "%s@%s" % (sid, str(uuid.uuid4()))
 
 
def create(self, annotation):
ANNOTATION_FILE_METADATA = LPATH['ANNOTATIONS'] + annotation['AID'] + '.json'
ANNOTATION_FILE_CONTENT = LPATH['ANNOTATIONS'] + annotation['AID'] + '.adoc'
annotation['Timestamp'] = self.srvutl.timestamp()
annotation['Created'] = self.srvutl.timestamp()
 
# Write annotation content first and delete it
with open(ANNOTATION_FILE_CONTENT, 'w') as fc:
fc.write(annotation['Content'])
del annotation['Content']
 
# Write annotation metadata
with open(ANNOTATION_FILE_METADATA, 'w') as fa:
json.dump(annotation, fa)
 
title = self.get_title(annotation['AID'])
self.log.info("Annotation '%s' (%s) created" % (title, annotation['AID']))
 
 
def update_metadata(self, metadata):
# Update annotation metadata
ANNOTATION_FILE_METADATA = LPATH['ANNOTATIONS'] + metadata['AID'] + '.json'
annotation = self.get_metadata_from_file(ANNOTATION_FILE_METADATA)
for key in metadata:
annotation[key] = metadata[key]
annotation['Timestamp'] = self.srvutl.timestamp()
# Write annotation metadata
with open(ANNOTATION_FILE_METADATA, 'w') as fa:
json.dump(annotation, fa)
 
title = self.get_title(annotation['AID'])
self.log.info("Annotation '%s' (%s) updated" % (title, annotation['AID']))
 
 
def update_timestamp(self, aid):
# Update annotation timestamp
ANNOTATION_FILE_METADATA = LPATH['ANNOTATIONS'] + aid + '.json'
annotation = self.get_metadata_from_file(ANNOTATION_FILE_METADATA)
annotation['Timestamp'] = self.srvutl.timestamp()
# Write annotation metadata
with open(ANNOTATION_FILE_METADATA, 'w') as fa:
json.dump(annotation, fa)
 
title = self.get_title(annotation['AID'])
self.log.info("Annotation '%s' (%s) updated" % (title, annotation['AID']))
def update(self, annotation):
ANNOTATION_FILE_METADATA = LPATH['ANNOTATIONS'] + annotation['AID'] + '.json'
ANNOTATION_FILE_CONTENT = LPATH['ANNOTATIONS'] + annotation['AID'] + '.adoc'
 
annotation['Timestamp'] = self.srvutl.timestamp()
# Write updated annotation first and delete the old one after
with open(ANNOTATION_FILE_CONTENT, 'w') as fc:
fc.write(annotation['Content'])
del annotation['Content']
 
# Write annotation metadata
with open(ANNOTATION_FILE_METADATA, 'w') as fa:
json.dump(annotation, fa)
 
title = self.get_title(annotation['AID'])
self.log.info("Annotation '%s' (%s) updated" % (title, annotation['AID']))
 
 
def delete(self, aid):
sid = self.get_sid(aid)
ANNOTATION_FILE_METADATA = LPATH['ANNOTATIONS'] + aid + '.json'
ANNOTATION_FILE_CONTENT = LPATH['ANNOTATIONS'] + aid + '.adoc'
title = self.get_title(aid)
 
if os.path.exists(ANNOTATION_FILE_METADATA):
os.unlink(ANNOTATION_FILE_METADATA)
 
if os.path.exists(ANNOTATION_FILE_CONTENT):
os.unlink(ANNOTATION_FILE_CONTENT)
 
self.log.info("Annotation '%s' (%s) deleted" % (title, aid))
 
 
def get_by_sid(self, sid):
ANNOTATION_FILES = LPATH['ANNOTATIONS'] + '%s*.json' % sid
annotations = glob.glob(ANNOTATION_FILES)
annotations.sort(reverse=True)
 
return annotations
 
 
def get_all(self):
return glob.glob(LPATH['ANNOTATIONS'] + '*.json')
 
 
def get_total(self):
return len(self.get_all())
 
 
def get_sid(self, aid):
if '@' in aid:
return aid[:aid.find('@')]
else:
return aid # aid = sid
 
 
def get_metadata_from_aid(self, aid=None):
if aid is not None:
ANNOTATION_FILE_METADATA = LPATH['ANNOTATIONS'] + aid + '.json'
with open(ANNOTATION_FILE_METADATA, 'r') as fa:
annotation = json.load(fa)
return annotation
else:
return None
 
 
def get_metadata_from_file(self, filename=None):
if filename is not None:
with open(filename, 'r') as fa:
annotation = json.load(fa)
return annotation
else:
return None
 
 
def is_valid(self, aid):
ANNOTATION_FILE = LPATH['ANNOTATIONS'] + aid + '.json'
valid = os.path.exists(ANNOTATION_FILE)
if valid is False:
self.log.debug("Annotation %s is not valid or it doesn't exist yet." % aid)
 
return valid
 
def get_title(self, aid):
ANNOTATION_FILE = LPATH['ANNOTATIONS'] + aid + '.json'
with open(ANNOTATION_FILE, 'r') as fa:
metadata = json.load(fa)
return metadata['Title']
 
 
def search_term(self, term):
matches = set()
annotations = self.get_all()
 
for fname in annotations:
# search only in title
with open(fname, 'r') as fa:
try:
annotation = json.load(fa)
text = annotation['Title']
if term.upper() in text.upper():
# ~ self.log.debug("Found '%s' in '%s'", term, text)
matches.add(fname)
except Exception as error:
self.log.error("%s: %s", fname, error)
 
# SEARCH IN ALL PROPERTIES (DISABLED)
# ~ with open(fname, 'r') as fa:
# ~ try:
# ~ annotation = json.load(fa)
# ~ text = ''
# ~ for node in annotation:
# ~ text += annotation[node]
# ~ if term.upper() in text.upper():
# ~ self.log.debug("Found '%s' in '%s'", term, text)
# ~ matches.add(fname)
# ~ except Exception as error:
# ~ self.log.error("%s: %s", fname, error)
 
# SEARCH IN CONTENT (DISABLED)
# ~ fcontent = fname.replace('.json', '.adoc')
# ~ text = open(fcontent, 'r').read()
# ~ if term.upper() in text.upper():
# ~ matches.add(fname)
 
return matches
 
 
def finalize(self):
pass
 
/branches/BR-0.4/basico/services/srv_collections.py
0,0 → 1,183
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
# File: srv_cols.py
# Author: Tomás Vírseda
# License: GPL v3
# Description: Collections service
"""
 
import os
import json
import uuid
 
from basico.core.mod_env import FILE
from basico.core.mod_srv import Service
 
COL_DOWNLOADED = "00000000-0000-0000-0000-000000000000"
 
HEADER = ['id', 'title']
 
class Collections(Service):
def initialize(self):
'''
Setup Collections Service
'''
self.get_services()
self.clts = {}
self.load_collections()
 
# Always check if 'Downloaded' collection exists
if not self.exists(COL_DOWNLOADED):
self.create('Downloaded', COL_DOWNLOADED)
 
 
def get_services(self):
self.srvgui = self.get_service("GUI")
self.srvuif = self.get_service("UIF")
self.srvclb = self.get_service('Callbacks')
self.srvicm = self.get_service('IM')
self.srvdtb = self.get_service('DB')
 
 
def exists(self, cid):
try:
self.clts[cid]
return True
except:
return False
 
 
def load_collections(self):
try:
with open(FILE['DBCOLS'], 'r') as ft:
self.clts = json.load(ft)
self.log.info("Collections Database found at: %s", FILE['DBCOLS'])
self.log.info ("Loaded %d collections" % len(self.clts))
except Exception as error:
self.save()
self.log.info("Collections database not found. Created a new database.")
 
 
def save(self, collections={}):
if len(collections) == 0:
collections = self.clts
with open(FILE['DBCOLS'], 'w') as ft:
json.dump(collections, ft)
self.log.debug ("Saved %d collections" % (len(collections)))
 
 
def get_all(self):
return self.clts
 
 
def get_collections_by_row_title(self, row_title):
cids = []
for cid in self.clts:
col_name = self.get_name_by_cid(cid)
if row_title in col_name:
cids.append(cid)
return cids
 
 
def get_collections_name(self):
names = []
for tid in self.clts:
names.append(self.clts[tid])
return names
 
 
def get_collections_id(self):
tids = []
for tid in self.clts:
tids.append(tid)
return tids
 
 
def create(self, name, cid=None, batch=False):
if len(name) == 0:
msg = "Trying to create a collection without name is not allowed"
self.log.debug(msg)
return (False, msg)
 
name_exists = name in self.get_collections_name()
if name_exists:
cid = self.get_cid_by_name(name)
msg = "Collection '%s' already exists in the database with cid: %s" % (name, cid)
self.log.info("Collection '%s' already exists in the database with cid: %s" % (name, cid))
return (False, "Collection '%s' already exists in the database" % name)
else:
if cid is None:
cid = str(uuid.uuid4())
cid_exists = cid in self.get_collections_id()
if cid_exists:
self.log.info("Collision? :) Let's try again with another id...")
self.create(name)
self.clts[cid] = name
msg = "Created collection: '%s' with cid: '%s'" % (name, cid)
self.log.info(msg)
if batch is False:
self.save()
return (True, "Created collection: %s" % name)
 
 
def delete(self, cid):
# Do not allow to delete this 'system' collection
name = self.get_name_by_cid(cid)
if name == 'Downloaded':
self.log.warning("You can't delete this collection")
return
try:
sapnotes = self.srvdtb.get_notes_by_node('collection', cid)
if len(sapnotes) > 0:
return False
else:
del(self.clts[cid])
msg = "Collection %s deleted" % name
self.log.info(msg)
self.save()
return True
except KeyError:
self.log.debug("You can't delete a non existent collection...")
except Exception as error:
self.log.debug("Error deleting collection: %s" % error)
self.log.debug(self.clts)
raise
 
 
def get_name_by_cid(self, cid):
try:
return self.clts[cid]
except KeyError:
return None
 
 
def get_cid_by_name(self, name):
database = self.get_all()
for col in database:
if database[col] == name:
return col
 
return None
 
 
def rename(self, cid, target):
# check if collections name are the same
source = self.get_name_by_cid(cid)
if source == target:
self.log.debug("Rename not possible: same name.")
return
 
# check if the new name already exists in database
acid = self.get_cid_by_name(target)
if acid is not None:
self.log.debug("Rename not possible: there is already another collection with this name.")
return
 
# Finally, rename the collection and save it.
self.clts[cid] = target
msg = "Collection '%s' renamed from '%s' to '%s'" % (cid, source, target)
self.log.debug(msg)
self.save()
/branches/BR-0.4/basico/services/srv_database.py
0,0 → 1,417
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
# File: srv_db.py
# Author: Tomás Vírseda
# License: GPL v3
# Description: Database service
"""
 
import os
from os.path import basename
import json
import glob
from html import escape
 
from basico.core.mod_env import LPATH, FILE
from basico.core.mod_srv import Service
from basico.services.srv_collections import COL_DOWNLOADED
 
SEP = os.path.sep
 
class Database(Service):
def initialize(self):
'''
Setup AppLogic Service
'''
self.sapnotes = {}
self.stats = {}
self.stats['maincomp'] = {}
self.stats['cats'] = {}
self.stats['component'] = set()
self.stats['category'] = set()
self.stats['priority'] = set()
self.stats['type'] = set()
self.stats['version'] = set()
self.__init_config_section()
self.get_services()
self.load_notes()
 
 
def __init_config_section(self):
settings = self.srvstg.load()
settings[self.section]
try:
settings[self.section]['DBSAP']
except:
settings[self.section]['DBSAP'] = FILE['DBSAP']
self.srvstg.save(settings)
 
def get_services(self):
self.srvclt = self.get_service('Collections')
 
def store(self, sapnote, html):
FSAPNOTE = LPATH['CACHE_XML'] + sapnote + '.xml'
 
try:
f = open(FSAPNOTE, 'w')
f.write(html)
f.close()
self.log.debug("SAP Note %s in XML format stored in %s" % (sapnote, FSAPNOTE))
except Exception as error:
self.log.error(error)
 
 
def get_sapnote_content(self, sid):
FSAPNOTE = LPATH['CACHE_XML'] + sid + '.xml'
content = open(FSAPNOTE, 'r').read()
return content
 
 
def is_saved(self, sid):
metadata = self.get_sapnote_metadata(sid)
if metadata is None:
return False
else:
return True
 
 
def is_valid(self, sid):
valid = True
for ch in sid:
if not ch.isdigit():
return False
return valid
 
 
def is_stored(self, sid):
fsapnote = LPATH['CACHE_XML'] + sid + '.xml'
stored = os.path.exists(fsapnote)
 
return stored
 
 
def build_stats(self, bag=None):
if bag is None:
bag = self.sapnotes
self.dstats = {}
self.compstats = {}
allcollections = set()
 
for sid in bag:
try:
collections = self.sapnotes[sid]['collections']
for collection in collections:
allcollections.add(collection)
except: pass
 
#COMMENT Components
compkey = self.sapnotes[sid]['componentkey']
comptxt = self.sapnotes[sid]['componenttxt']
component = escape("%s (%s)" % (compkey, comptxt))
sep = compkey.find('-')
if sep > 0:
maincomp = compkey[0:sep]
else:
maincomp = compkey
 
#COMMENT Categories
category = escape(self.sapnotes[sid]['category'])
try:
cont = self.stats['cats'][category]
self.stats['cats'][category] = cont + 1
except:
self.stats['cats'][category] = 1
 
#COMMENT Build all (sub)keys from given component key
#COMMENT useful for display stats with pygal
compkeys = compkey.split('-')
total = len(compkeys)
key = ''
i = 0
for subkey in compkeys:
key = key + '-' + subkey
if key[:1] == '-':
key = key[1:]
 
# update stats
try:
count = self.compstats[key]
self.compstats[key] = count + 1
except Exception as error:
self.compstats[key] = 1
 
try:
cont = self.stats['maincomp'][maincomp]
self.stats['maincomp'][maincomp] = cont + 1
except:
self.stats['maincomp'][maincomp] = 1
 
category = escape(self.sapnotes[sid]['category'])
priority = escape(self.sapnotes[sid]['priority'])
ntype = escape(self.sapnotes[sid]['type'])
version = escape(self.sapnotes[sid]['version'])
releaseon = escape(self.sapnotes[sid]['releasedon'])
self.stats['component'].add(component)
self.stats['category'].add(category)
self.stats['priority'].add(priority)
self.stats['type'].add(ntype)
self.stats['version'].add(version)
#FIXME self.stats['releaseon'].add(releasedon)
#FIXME self.stats[''].add(version)
 
 
def add(self, sapnote, overwrite=False, batch=False):
sid = sapnote['id']
 
if self.exists(sid):
if overwrite:
self.sapnotes[sid] = sapnote
self.log.info("SAP Note %s added to database (Overwrite is %s)", sid, overwrite)
if batch is False:
self.save_notes()
return True
else:
self.log.info("SAP Note %s already exists in database (Overwrite is %s)", sid, overwrite)
return False
else:
self.sapnotes[sid] = sapnote
self.log.info("SAP Note %s added to database (Overwrite is %s)", sid, overwrite)
if batch is False:
self.save_notes()
return True
 
 
def add_list(self, sapnotes, overwrite=False):
self.log.info("Add %d SAP Notes to database (Overwrite is %s)", len(sapnotes), overwrite)
n = 0
for sid in sapnotes:
if overwrite:
if self.add(sapnotes[sid], overwrite, batch=True):
n += 1
self.save_notes()
return n
 
def get_notes(self):
'''
Return all sapnotes
'''
return self.sapnotes
 
 
def get_notes_by_node(self, key, value):
'''
Return a set of sapnotes which matches with an specific key/value
'''
bag = set()
sapnotes = self.get_notes()
for sapnote in sapnotes:
if key.startswith('component') or \
key in ['category', 'priority', 'type']:
if sapnotes[sapnote][key].startswith(value):
bag.add(sapnote)
elif key == 'collection':
try:
collections = sapnotes[sapnote]['collections']
if value == 'None':
if len(collections) == 0:
bag.add(sapnote)
else:
if value in collections:
bag.add(sapnote)
except:
pass
elif key in ['date-year', 'date-month', 'date-day']:
if len(value) == 6:
value = value[0:4] + '-' + value[4:6]
elif len(value) == 8:
value = value[0:4] + '-' + value[4:6] + '-' + value[6:8]
 
if sapnotes[sapnote]['feedupdate'].startswith(value):
bag.add(sapnote)
else:
pass
 
return bag
 
 
def get_total(self):
'''
Return total sapnotes
'''
return len(self.sapnotes)
 
 
def load_notes(self):
'''
If notes.json exists, load notes
'''
try:
with open(FILE['DBSAP'], 'r') as fp:
self.sapnotes = json.load(fp)
self.log.info("SAP Notes Database found at: %s", FILE['DBSAP'])
self.log.info("Loaded %d notes from SAP Notes database" % len(self.sapnotes))
except Exception as error:
self.save_notes()
self.log.info("SAP Notes database not found. Created a new database for SAP Notes")
 
 
def get_sapnote_metadata(self, sid):
sid = self.normalize_sid(sid)
 
try:
return self.sapnotes[sid]
except KeyError as error:
# ~ self.log.debug("SAP Note %s doesn't exist in the database or it's been deleted" % sid)
return None
 
 
def exists(self, sid):
sid = self.normalize_sid(sid)
metadata = self.get_sapnote_metadata(sid)
if metadata is not None:
return True
else:
return False
 
def get_title(self, sid):
title = ''
metadata = self.get_sapnote_metadata(sid)
if metadata is not None:
title = metadata['title']
 
return title
 
 
def get_component(self, sid):
component = ''
metadata = self.get_sapnote_metadata(sid)
if metadata is not None:
component = metadata['component-key']
 
return component
 
 
def save_notes(self):
'''
Save SAP Notes to json database file
'''
 
bag = self.get_notes()
 
try:
with open(FILE['DBSAP'], 'w') as fdb:
json.dump(bag, fdb)
self.log.debug ("Saved %d notes to %s" % (len(bag), FILE['DBSAP']))
except Exception as error:
self.log.error(error)
 
 
def set_bookmark(self, lsid):
for sid in lsid:
sid = self.normalize_sid(sid)
self.sapnotes[sid]['bookmark'] = True
self.log.info("SAP Note %s bookmarked: True" % sid)
self.save_notes()
 
 
def set_no_bookmark(self, lsid):
for sid in lsid:
sid = self.normalize_sid(sid)
self.sapnotes[sid]['bookmark'] = False
self.log.info("SAP Note %s bookmarked: False" % sid)
self.save_notes()
 
 
def is_bookmark(self, sapnote):
try:
return self.sapnotes[sapnote]['bookmark']
except:
return False
 
def get_collections(self, sid):
try:
return self.sapnotes[sid]['collections']
except:
return []
 
 
def get_stats(self):
return self.stats
 
 
def search(self, term):
bag = []
 
for sid in self.get_notes():
values = []
sapnote = self.sapnotes[sid]
for key in sapnote:
values.append(str(sapnote[key]))
text = ' '.join(values)
if term.upper() in text.upper():
bag.append(sapnote['id'])
self.log.debug("Found term '%s' in %d SAP Notes" % (term, len(bag)))
return bag
 
 
def normalize_sid(self, sid):
if isinstance(sid, int):
sid = "0"*(10 - len(str(sid))) + str(sid)
elif isinstance(sid, str):
sid = "0"*(10 - len(sid)) + sid
 
return sid
 
 
def set_collections(self, sid, collections, overwrite=False):
sid = self.normalize_sid(sid)
self.log.info("SAP Note %s added to the following collections (overwrite mode is %s):", sid, overwrite)
if sid != '0000000000':
sid = self.normalize_sid(sid)
try:
if overwrite:
new_cols = []
self.sapnotes[sid]['collections'] = collections
for cid in collections:
if cid == COL_DOWNLOADED:
continue
if cid not in new_cols:
new_cols.append(cid)
self.log.info("\tCollection: %s", self.srvclt.get_name_by_cid(cid))
else:
current_collections = self.sapnotes[sid]['collections']
current_collections.extend(collections)
bag = []
for cid in current_collections:
if cid == COL_DOWNLOADED:
continue
if cid not in bag:
bag.append(cid)
self.log.info("\tCollection: %s", self.srvclt.get_name_by_cid(cid))
self.sapnotes[sid]['collections'] = bag
self.save_notes()
except:
self.log.error(error)
 
def delete(self, sid):
sid = self.normalize_sid(sid)
deleted = False
try:
del (self.sapnotes[sid])
deleted = True
self.log.info("SAP Note %s deleted" % sid)
self.save_notes()
except Exception as error:
self.log.debug(error)
deleted = False
 
return deleted
 
 
def end(self):
self.save_notes()
 
/branches/BR-0.4/basico/services/srv_sap.py
12,7 → 12,7
from shutil import which
from basico.core.mod_srv import Service
from basico.core.mod_env import LPATH
from basico.services.srv_cols import COL_DOWNLOADED
from basico.services.srv_collections import COL_DOWNLOADED
 
# Default settings for SAP module
LOGIN_PAGE_URL = "https://accounts.sap.com"
/branches/BR-0.4/basico/widgets/wdg_cols.py
12,7 → 12,7
from gi.repository import Gtk
from gi.repository import GObject
from basico.core.mod_wdg import BasicoWidget
from basico.services.srv_cols import COL_DOWNLOADED
from basico.services.srv_collections import COL_DOWNLOADED
 
 
class CollectionsMgtView(BasicoWidget, Gtk.VBox):
/branches/BR-0.4/basico/widgets/wdg_menuview.py
21,7 → 21,7
from dateutil import parser as dateparser
from basico.widgets.wdg_cols import CollectionsMgtView
from basico.core.mod_wdg import BasicoWidget
from basico.services.srv_cols import COL_DOWNLOADED
from basico.services.srv_collections import COL_DOWNLOADED
 
class Row(IntEnum):
ROWID = 0
/branches/BR-0.4/basico/widgets/wdg_visor_sapnotes.py
25,7 → 25,7
 
from basico.core.mod_wdg import BasicoWidget
from basico.core.mod_env import LPATH, ATYPES
from basico.services.srv_cols import COL_DOWNLOADED
from basico.services.srv_collections import COL_DOWNLOADED
from basico.widgets.wdg_cols import CollectionsMgtView
from basico.widgets.wdg_import import ImportWidget
from basico.core.mod_log import get_logger