[Uludag-commits] r15084 - in trunk/staj-projeleri/buildfarm: . helpers
uludag-commits at pardus.org.tr
uludag-commits at pardus.org.tr
23 Ağu 2007 Per 09:43:03 EEST
Author: ozan.caglayan
Date: Thu Aug 23 09:43:03 2007
New Revision: 15084
Modified:
trunk/staj-projeleri/buildfarm/helpers/qmanager.py
trunk/staj-projeleri/buildfarm/helpers/repomanager.py
trunk/staj-projeleri/buildfarm/server.py
Log:
code refactored,added sync()
Modified: trunk/staj-projeleri/buildfarm/helpers/qmanager.py
=================================================================
--- trunk/staj-projeleri/buildfarm/helpers/qmanager.py (original)
+++ trunk/staj-projeleri/buildfarm/helpers/qmanager.py Thu Aug 23 09:43:03 2007
@@ -12,10 +12,24 @@
""" Standart Python Modules """
import os
+import sys
+
+from shutil import copy as shutilCopy
+from copy import copy as shallowCopy
""" BuildFarm Modules """
import config
import dependency
+import logger
+import mailer
+
+""" Helpers """
+from helpers import pisiinterface
+
+""" Gettext Support """
+import gettext
+__trans = gettext.translation("buildfarm", fallback = True)
+_ = __trans.ugettext
class QueueManager:
def __init__(self):
@@ -119,3 +133,183 @@
self.removeFromWorkQueue(pspec)
return True
return False
+
+ def buildPackages(self):
+
+ sys.excepthook = self.__handle_exception__
+
+ def createDirectories():
+ directories = [config.workDir,
+ config.binaryPath,
+ config.localPspecRepo,
+ config.outputDir]
+
+ for dir in directories:
+ if dir and not os.path.isdir(dir):
+ try:
+ os.makedirs(dir)
+ except OSError:
+ raise _("Directory '%s' cannot be created, permission problem?") % dir
+
+ # Make sure that directories exists.
+ createDirectories()
+
+ queue = shallowCopy(self.getWorkQueue())
+
+ if len(queue) == 0:
+ logger.info(_("Both Wait and Work Queues are empty..."))
+ return True
+
+ logger.raw(_("QUEUE"))
+ logger.info(_("Work Queue: %s") % (self.getWorkQueue()))
+ sortedQueue = self.getWorkQueue()[:]
+ sortedQueue.sort()
+ # mailer.info(_("I'm starting to compile following packages:\n\n%s") % "\n".join(sortedQueue))
+ logger.raw()
+
+ for pspec in queue:
+ packagename = os.path.basename(os.path.dirname(pspec))
+ build_output = open(os.path.join(config.outputDir, "%s.log" % packagename), "w")
+ logger.info(
+ _("Compiling source %s (%d of %d)") %
+ (
+ packagename,
+ int(queue.index(pspec) + 1),
+ len(queue)
+ )
+ )
+ logger.raw()
+
+ pisi = pisiinterface.PisiApi(config.workDir)
+ pisi.init(stdout = build_output, stderr = build_output)
+ try:
+ try:
+ (newBinaryPackages, oldBinaryPackages) = pisi.build(pspec)
+ except Exception, e:
+ self.transferToWaitQueue(pspec)
+ errmsg = _("Error occured for '%s' in BUILD process:\n %s") % (pspec, e)
+ logger.error(errmsg)
+ # mailer.error(errmsg, pspec)
+ else:
+ try:
+ for p in newBinaryPackages:
+ logger.info(_("Installing: %s" % os.path.join(config.workDir, p)))
+ pisi.install(os.path.join(config.workDir, p))
+ except Exception, e:
+ self.transferToWaitQueue(pspec)
+ errmsg = _("Error occured for '%s' in INSTALL process: %s") % (os.path.join(config.workDir, p), e)
+ logger.error(errmsg)
+ # mailer.error(errmsg, pspec)
+
+ newBinaryPackages.remove(p)
+ self.__removeBinaryPackageFromWorkDir__(p)
+ else:
+ self.removeFromWorkQueue(pspec)
+ self.__movePackages__(newBinaryPackages, oldBinaryPackages)
+ finally:
+ pisi.finalize()
+
+ logger.raw(_("QUEUE"))
+ logger.info(_("Wait Queue: %s") % (self.getWaitQueue()))
+ logger.info(_("Work Queue: %s") % (self.getWorkQueue()))
+
+ if self.getWaitQueue():
+ # mailer.info(_("Queue finished with problems and those packages couldn't be compiled:\n\n%s\n") % "\n".join(qmgr.getWaitQueue()))
+ return self.getWaitQueue()
+ else:
+ # mailer.info(_("Queue finished without a problem!..."))
+ pass
+ return True
+
+ def buildIndex(self):
+ logger.raw()
+ logger.info(_("Generating PiSi Index..."))
+
+ current = os.getcwd()
+ os.chdir(config.binaryPath)
+ os.system("/usr/bin/pisi index %s . --skip-signing --skip-sources" % config.localPspecRepo)
+ logger.info(_("PiSi Index generated..."))
+
+ #FIXME: will be enabled after some internal tests
+ #os.system("rsync -avze ssh --delete . pisi.pardus.org.tr:/var/www/paketler.uludag.org.tr/htdocs/pardus-1.1/")
+
+ # Check packages containing binaries and libraries broken by any package update
+ os.system("/usr/bin/revdep-rebuild --force")
+ # FIXME: if there is any broken package, mail /root/.revdep-rebuild.4_names file
+
+ os.chdir(current)
+
+ # FIXME: handle indexing errors
+ return True
+
+ def __movePackages__(self, newBinaryPackages, oldBinaryPackages):
+ # sanitaze input
+ try:
+ newBinaryPackages = set(map(lambda x: os.path.basename(x), newBinaryPackages))
+ except AttributeError:
+ pass
+
+ try:
+ oldBinaryPackages = set(map(lambda x: os.path.basename(x), oldBinaryPackages))
+ except AttributeError:
+ pass
+
+ unchangedPackages = set(newBinaryPackages).intersection(set(oldBinaryPackages))
+ newPackages = set(newBinaryPackages) - set(oldBinaryPackages)
+ oldPackages = set(oldBinaryPackages) - set(unchangedPackages)
+
+ logger.info(_("*** New binary package(s): %s") % newPackages)
+ logger.info(_("*** Old binary package(s): %s") % oldPackages)
+ logger.info(_("*** Unchanged binary package(s): %s") % unchangedPackages)
+
+ exists = os.path.exists
+ join = os.path.join
+ remove = os.remove
+ copy = shutilCopy
+
+ def __moveOldPackage__(self, package):
+ logger.info(_("*** Old package '%s' is processing") % (package))
+ if exists(join(config.binaryPath, package)):
+ remove(join(config.binaryPath, package))
+
+ if exists(join(config.workDir, package)):
+ remove(join(config.workDir, package))
+
+ def __moveNewPackage__(self, package):
+ logger.info(_("*** New package '%s' is processing") % (package))
+ if exists(join(config.workDir, package)):
+ copy(join(config.workDir, package), config.binaryPath)
+ remove(join(config.workDir, package))
+
+ def __moveUnchangedPackage__(self, package):
+ logger.info(_("*** Unchanged package '%s' is processing") % (package))
+ if exists(join(config.workDir, package)):
+ copy(join(config.workDir, package), config.binaryPath)
+ remove(join(config.workDir, package))
+
+ for package in newPackages:
+ if package:
+ __moveNewPackage__(self, package)
+
+ for package in oldPackages:
+ if package:
+ __moveOldPackage__(self, package)
+
+ for package in unchangedPackages:
+ if package:
+ __moveUnchangedPackage__(self, package)
+
+ def __removeBinaryPackageFromWorkDir__(self, package):
+ join = os.path.join
+ remove = os.remove
+ remove(join(config.workDir, package))
+
+ def __handle_exception__(self, exception, value, tb):
+ s = cStringIO.StringIO()
+ traceback.print_tb(tb, file = s)
+ s.seek(0)
+
+ logger.error(str(exception))
+ logger.error(str(value))
+ logger.error(s.read())
+
Modified: trunk/staj-projeleri/buildfarm/helpers/repomanager.py
=================================================================
--- trunk/staj-projeleri/buildfarm/helpers/repomanager.py (original)
+++ trunk/staj-projeleri/buildfarm/helpers/repomanager.py Thu Aug 23 09:43:03 2007
@@ -103,3 +103,30 @@
queue.write("%s\n" % pspec)
queue.close()
return updatedpspecfiles + newpspecfiles
+
+ # configure source and dest
+ def sync(self, source="./pardus-2007-test", dest="./pardus-2007", sendMail=False):
+
+ diff = []
+ for i in os.listdir(source):
+ if not os.path.exists(dest + "/" + i):
+ # os.system("cp %s/%s %s/%s" % (source, i, dest, i))
+ # os.system("pisi index /root/2007 pardus-2007 --skip-signing --skip-sources")
+ diff.append(i)
+
+ diff.sort()
+
+ if sendMail:
+ # send "[RFC] Güncellemeler hedere hodoro" to the list
+ pass
+
+ return diff
+
+ #import os
+ #
+ #for i in os.listdir("pardus-2007-test/"):
+ # if not os.path.exists("pardus-2007/%s" % i):
+ # # print "copying pardus-2007-test/%s --> pardus-2007/%s" % (i, i)
+ # #os.system("cp pardus-2007-test/%s pardus-2007/%s" % (i, i))
+ # #os.system("pisi index /root/2007 pardus-2007 --skip-signing --skip-sources")
+ # print i
Modified: trunk/staj-projeleri/buildfarm/server.py
=================================================================
--- trunk/staj-projeleri/buildfarm/server.py (original)
+++ trunk/staj-projeleri/buildfarm/server.py Thu Aug 23 09:43:03 2007
@@ -27,11 +27,10 @@
""" pyOpenSSL """
from OpenSSL import SSL
-""" helpers """
-from helpers import qmanager
+""" Helpers """
+from helpers import qmanager2 as qmanager
from helpers import repomanager
-import main
import config
# Inherits from ForkingMixIn for multi-process support
@@ -117,11 +116,6 @@
# export CombinedServerClass
server.register_instance(CombinedServerClass())
- # export buildPackages, buildIndex
- # FIXME: run these on another thread and return to client ASAP
- server.register_function(main.buildPackages)
- server.register_function(main.buildIndex)
-
# enter main loop
server.serve_forever()
Uludag-commits mesaj listesiyle ilgili
daha fazla bilgi