Classes | |
| class | qoid_fixer |
Functions | |
| def | find_XML |
| def | diff_fixes |
| def | catalog_files |
| def | catalog_files_version |
| def | create_cache |
| def | remove_entity |
| def | merge_xml |
| def | sort_by_dependencies |
| def | merge_xml_files |
| def | cleanup_xml |
| def | sync_with_xml |
| def | build_from_xml |
| def | available_updates |
| def | update |
| def | fetch_cover |
Variables | |
| dictionary | model_from_xml |
| tuple | models = model_from_xml.values() |
| version | |
| def py::qo::catalog::available_updates | ( | ) |
check for available catalog updates
00291 : 00292 """ 00293 check for available catalog updates 00294 """ 00295 def __filter_fix( name ) : 00296 return name.startswith( "fix_" ) 00297 (version,is_obsolete,files,fixes) = qo.network.read_XMLIndex( qocfg.get_updateXML_index(), qocfg.mirrors_update) 00298 xml_mod = find_XML( files, catalog_files_version() ) 00299 fix_mod = diff_fixes(fixes, filter(__filter_fix, os.listdir(qocfg.xml_dir))) 00300 return xml_mod, fix_mod 00301 00302 00303 00304 class qoid_fixer :
| def py::qo::catalog::build_from_xml | ( | progress | ) |
build entire catalog from xml files
00284 : 00285 """ 00286 build entire catalog from xml files 00287 """ 00288 merge_xml_files(catalog_files(), progress) 00289 cleanup_xml(progress) 00290 def available_updates() :
| def py::qo::catalog::catalog_files | ( | ) |
00064 : 00065 def is_xml(f) : 00066 return f.endswith(".xml") 00067 files = [] 00068 for xml_dir in qocfg.get_xml_dirs() : 00069 files += map(lambda x: os.path.join(xml_dir,x), filter(is_xml,os.listdir(xml_dir))) 00070 return files 00071 def catalog_files_version() :
| def py::qo::catalog::catalog_files_version | ( | ) |
00072 : 00073 00074 class parsing_done( Exception ) : 00075 pass 00076 00077 class extract_version(xml.sax.handler.ContentHandler) : 00078 def startElement( self, name, attrs ) : 00079 self.version = attrs.getValue("version") 00080 raise parsing_done 00081 00082 parser = xml.sax.make_parser() 00083 parser.setFeature(xml.sax.handler.feature_external_ges, False) 00084 parse_version = extract_version() 00085 parser.setContentHandler(parse_version) 00086 00087 files = {} 00088 for xml_dir in qocfg.get_xml_dirs() : 00089 for filename in os.listdir(xml_dir) : 00090 if filename[-4:] == ".xml": 00091 path = os.path.join(xml_dir,filename) 00092 name = os.path.basename(filename)[:-4] 00093 try : 00094 parser.parse(path) 00095 except parsing_done : 00096 files[name] = {"name" : name, "version" : parse_version.version, "path" : path } 00097 return files 00098 model_from_xml = {
| def py::qo::catalog::cleanup_xml | ( | progress | ) |
00254 : 00255 all = [] 00256 for path in catalog_files() : 00257 all.append(qo.utils.file_id(path)) 00258 for catalog in qodb.models.CatalogVersion.objects.exclude(name__in=all) : 00259 d = {} 00260 catalog._collect_sub_objects(d) 00261 for e in [e for k, e in qo.utils.listflatten([i.items() for i in d.itervalues()]) if e.__class__ in models] : 00262 if not remove_entity(e) : 00263 e.catalog = None 00264 e.created_by_user = True 00265 if e.has_underlay() : 00266 e._underlay.delete() 00267 e.save() 00268 catalog.delete() 00269 def sync_with_xml( progress ) :
| def py::qo::catalog::create_cache | ( | ) |
00108 : 00109 return { 00110 "editor" : {}, 00111 "collection" : {}, 00112 "series" : {}, 00113 "authors" : {}, 00114 } 00115 def remove_entity( e ) :
| def py::qo::catalog::diff_fixes | ( | fix_dict, | ||
| user_list | ||||
| ) |
00061 : 00062 return [], [f for f in fix_dict if f not in user_list], [f for f in user_list if f not in fix_dict] 00063 def catalog_files() :
| def py::qo::catalog::fetch_cover | ( | name | ) |
| def py::qo::catalog::find_XML | ( | update_dict, | ||
| user_dict | ||||
| ) |
00039 : 00040 new = [] 00041 remove = [] 00042 update = [] 00043 00044 if user_dict is None : 00045 user_dict = {} 00046 00047 user_files = user_dict.keys() 00048 for f in user_files : 00049 if not update_dict.has_key( f ) : 00050 remove.append( user_dict[ f ] ) 00051 del user_dict[ f ] 00052 00053 for f in update_dict.keys() : 00054 if not user_dict.has_key( f ) : 00055 new.append( update_dict[ f ] ) 00056 elif qo.utils.is_version_newer( update_dict[ f ][ 'version' ], user_dict[ f ][ 'version' ] ) : 00057 update.append( update_dict[ f ] ) 00058 00059 return ( update, new, remove ) 00060 def diff_fixes( fix_dict, user_list ) :
| def py::qo::catalog::merge_xml | ( | filename, | ||
cache = None | ||||
| ) |
merge xml filename into catalog
00138 : 00139 """ 00140 merge xml filename into catalog 00141 """ 00142 doc = xml.dom.minidom.parse(filename) 00143 root = doc.documentElement 00144 00145 name = ".".join(os.path.basename(filename).split('.')[:-1]) 00146 version = root.getAttribute("version") 00147 00148 catalog, is_new = qodb.models.CatalogVersion.objects.get_or_create(name=name) 00149 00150 if not cache : 00151 cache = create_cache() 00152 00153 all = { "AUTHOR" : [], "EDITOR" : [], "COLLECTION" : [], "ASERIES" : [], "COMIC" : [] } 00154 00155 for node in root.childNodes : 00156 if node.nodeType is not xml.dom.Node.ELEMENT_NODE : 00157 continue 00158 t = node.localName 00159 00160 model = model_from_xml[t] 00161 00162 try : 00163 obj = model.objects.get(qoid=node.getAttribute("id")) 00164 except model.DoesNotExist : 00165 obj = model(qoid=node.getAttribute("id")) 00166 00167 # common attrs 00168 attrs = { 00169 "name" : node.getAttribute("name"), 00170 "catalog" : catalog, 00171 } 00172 if t == 'AUTHOR' : 00173 cache["authors"][obj.qoid] = obj 00174 elif t == 'EDITOR' : 00175 attrs["homepage"] = node.getAttribute("web_uri") 00176 cache["editor"][obj.qoid] = obj 00177 elif t == 'COLLECTION' : 00178 editor_qoid = node.getAttribute("editor") 00179 if editor_qoid not in cache["editor"] : 00180 cache["editor"][editor_qoid] = qodb.models.Editor.objects.get(qoid=editor_qoid) 00181 attrs["editor"] = cache["editor"][editor_qoid] 00182 cache["collection"][obj.qoid] = obj 00183 elif t == 'ASERIES' : 00184 collection_qoid = node.getAttribute("collection") 00185 if collection_qoid not in cache["collection"] : 00186 cache["collection"][collection_qoid] = qodb.models.Collection.objects.get(qoid=collection_qoid) 00187 attrs["collection"] = cache["collection"][collection_qoid] 00188 cache["series"][obj.qoid] = obj 00189 elif t == 'COMIC' : 00190 series_qoid = node.getAttribute("series") 00191 if series_qoid not in cache["series"] : 00192 cache["series"][series_qoid] = qodb.models.Series.objects.get(qoid=series_qoid) 00193 attrs["series"] = cache["series"][series_qoid] 00194 attrs["volume"] = int(node.getAttribute("volume") or "0") 00195 attrs["authors"] = sorted(node.getAttribute("author").split()) 00196 attrs["drawers"] = sorted(node.getAttribute("drawer").split()) 00197 attrs["published_date"] = node.getAttribute("date") 00198 attrs["isbn"] = node.getAttribute("isbn") 00199 attrs["ean"] = node.getAttribute("ean") 00200 attrs["guiding_price"] = float(node.getAttribute("guiding_price") or .0) 00201 obj.underlay(cache=cache, **attrs) 00202 all[t].append(obj.qoid) 00203 00204 for t, ids in all.iteritems() : 00205 if len(ids) > 998 : 00206 # XXX django/sqlite bug 00207 rids = ids[998:] 00208 ids = ids[:998] 00209 map(remove_entity, [e for e in model_from_xml[t].objects.exclude(qoid__in=ids).filter(catalog=catalog) if e.qoid not in rids]) 00210 else : 00211 map(remove_entity, model_from_xml[t].objects.exclude(qoid__in=ids).filter(catalog=catalog)) 00212 00213 catalog.version = version 00214 catalog.save() 00215 00216 def sort_by_dependencies( files, getter=None ) :
| def py::qo::catalog::merge_xml_files | ( | files, | ||
| progress | ||||
| ) |
00246 : 00247 cache = create_cache() 00248 files = sort_by_dependencies(files) 00249 progress.section_grow(len(files)) 00250 for file in files : 00251 progress.step(_("Merging %s")%os.path.basename(file)) 00252 merge_xml(file, cache) 00253 def cleanup_xml( progress ) :
| def py::qo::catalog::remove_entity | ( | e | ) |
remove entity only if nothing (except underlay) depend on it returns True if the entity has been removed, False else
00116 : 00117 """ 00118 remove entity only if nothing (except underlay) depend on it 00119 returns True if the entity has been removed, False else 00120 """ 00121 assert e.__class__ in models 00122 for related in [r for r in e._meta.get_all_related_objects() if r.model is not e.get_underlay_class()] : 00123 r_field = related.get_accessor_name() 00124 if isinstance(related.field.rel, django.db.models.fields.related.OneToOneRel) : 00125 try : 00126 getattr(e, r_field) 00127 return False 00128 except django.core.exceptions.ObjectDoesNotExist : 00129 pass 00130 else : 00131 if len(getattr(e, r_field).all()) > 0 : 00132 return False 00133 e.delete() 00134 return True 00135 00136 00137 @qodb.transaction.commit_on_success def merge_xml( filename, cache=None ) :
| def py::qo::catalog::sort_by_dependencies | ( | files, | ||
getter = None | ||||
| ) |
return sorted files by dependencies
00217 : 00218 """ 00219 return sorted files by dependencies 00220 """ 00221 order = [ "author", "authors", "editor", "editors", "collection", "collections", "series", "comics" ] 00222 def get_type( f ) : 00223 return os.path.basename(f).replace('.','_').split('_')[0] 00224 def cmp_depend( f1, f2 ) : 00225 if getter : 00226 f1 = getter(f1) 00227 f2 = getter(f2) 00228 i1, i2 = order.index(get_type(f1)), order.index(get_type(f2)) 00229 if i1 != i2 : 00230 return i1 - i2 00231 else : 00232 return cmp(os.path.basename(f1),os.path.basename(f2)) 00233 def ensure_type( filename ) : 00234 if getter : 00235 filename = getter(filename) 00236 ft = get_type(filename) 00237 if ft not in order : 00238 qo_error(_("Warning: '%s' is not a catalog file!")%filename) 00239 return False 00240 return True 00241 files = filter(ensure_type, files) 00242 return sorted(files,cmp_depend) 00243 00244 00245 @qodb.transaction.commit_on_success def merge_xml_files( files, progress ) :
| def py::qo::catalog::sync_with_xml | ( | progress | ) |
merge xml files with newer version
00270 : 00271 """ 00272 merge xml files with newer version 00273 """ 00274 todo = [] 00275 cur = {} 00276 for o in qodb.models.CatalogVersion.objects.all() : 00277 cur[o.name] = o.version 00278 for d in catalog_files_version().itervalues() : 00279 if d["name"] not in cur or qo.utils.is_version_newer(d["version"], cur[d["name"]]) : 00280 todo.append(d["path"]) 00281 merge_xml_files(todo, progress) 00282 cleanup_xml(progress) 00283 def build_from_xml( progress ) :
| def py::qo::catalog::update | ( | progress | ) |
apply available catalog updates
00344 : 00345 """ 00346 apply available catalog updates 00347 """ 00348 progress.grow(4) 00349 00350 progress.section(_("Read available updates"),1) 00351 progress.step(_("Fetching catalog index")) 00352 00353 m_files, m_fixes = available_updates() 00354 up_files = sort_by_dependencies(m_files[0] + m_files[1], operator.itemgetter('name')) 00355 up_fixes = m_fixes[0] + m_fixes[1] 00356 del_files = m_files[2] 00357 del_fixes = m_fixes[2] 00358 00359 progress.section_end() 00360 00361 progress.section(_("Fetching catalog updates"), len(up_files)) 00362 for d in up_files : 00363 progress.step(d["name"]) 00364 qo.network.gzretrieve_from_mirror(qocfg.mirrors_update, d["src"], os.path.join(qocfg.xml_dir, "%s.xml"%d['name'])) 00365 try : 00366 qo.sh.rmf(*[os.path.join(qocfg.xml_dir, "%s.xml" % d) for d in del_files]) 00367 except : 00368 traceback.print_exc() 00369 progress.section_end() 00370 00371 progress.section(_("Fixes"), 2) 00372 progress.step(_("Fetching...")) 00373 for d in up_fixes : 00374 qo.network.retrieve_from_mirror(qocfg.mirrors_update, d["src"], os.path.join(qocfg.xml_dir, d['name'])) 00375 try : 00376 qo.sh.rmf(*[os.path.join(qocfg.xml_dir, d) for d in del_fixes]) 00377 except : 00378 traceback.print_exc() 00379 progress.step(_("Applying...")) 00380 00381 qoid_fixer.load() 00382 fixer = qoid_fixer() 00383 if len(fixer) : 00384 for orig in qodb.models.Album.objects.filter(qoid__in=list(fixer.oldies())) : 00385 print "fixing", orig 00386 dest_qoid = fixer[orig.qoid] 00387 # move data 00388 for get_path in [qocfg.get_cover_path, qocfg.get_comment_path] : 00389 orig_path = get_path(orig.qoid) 00390 dest_path = get_path(dest_qoid) 00391 if os.path.exists(orig_path) and not os.path.exists(dest_path) : 00392 shutil.move(orig_path, dest_path) 00393 for copy in orig.albumcopy_set.all() : 00394 orig_path = qocfg.get_copy_comment_path(orig.qoid, copy.id) 00395 dest_path = qocfg.get_copy_comment_path(dest_qoid, copy.id) 00396 if os.path.exists(orig_path) and not os.path.exists(dest_path) : 00397 shutil.move(orig_path, dest_path) 00398 # move db 00399 try : 00400 dest = qodb.models.Album.objects.get(qoid=dest_qoid) 00401 for copy in orig.albumcopy_set.all() : 00402 copy.original_edition = dest 00403 copy.save() 00404 orig.delete() 00405 except qodb.models.Album.DoesNotExist : 00406 orig.qoid = dest_qoid 00407 orig.save() 00408 00409 progress.section_end() 00410 00411 progress.section(_("Applying catalog updates"), len(up_files)) 00412 sync_with_xml(progress) 00413 progress.section_end() 00414 00415 def fetch_cover( name ) :
dictionary py::qo::catalog::model_from_xml [static] |
Initial value:
{
"AUTHOR" : qodb.models.Author,
"EDITOR" : qodb.models.Editor,
"COLLECTION" : qodb.models.Collection,
"ASERIES" : qodb.models.Series,
"COMIC" : qodb.models.Album,
}
tuple py::qo::catalog::models = model_from_xml.values() [static] |
1.5.3