streamline build scripts

releng.control/cc.sh

@@ -9,8 +9,9 @@ then
    source ${BUILD_HOME}/releng.control/commonComputedVariables.shsource  
 fi
 
-port="7000"
+jmxport="7000"
 webport="7777"
+rmiport="1099"
 # its ok for these to be trivial, just used to prevent 
 # accidental use, no real security needed. 
 trivialUserName="wtp"
@@ -23,5 +24,5 @@ CCNAME="Webtools"
 # We want to execute CC itself in Java 5
 export JAVA_HOME=${JAVA_5_HOME}
 
-sh ${CCDIR}/cruisecontrol.sh -configfile $BUILD_HOME/releng.control/cc_config.xml -jmxport $port -webport $webport -user $trivialUserName -password $trivialPw -cchome ${CCDIR} -ccname ${CCNAME} 1>out.txt 2>err.txt &
+sh ${CCDIR}/cruisecontrol.sh -configfile ${RELENG_CONTROL}/cc_config.xml -jmxport $jmxport -rmiport $rmiport -webport $webport -user $trivialUserName -password $trivialPw -cchome ${CCDIR} -ccname ${CCNAME} 1>out.txt 2>err.txt &
 

releng.control/cc_config.xml

@@ -37,14 +37,14 @@
         file="${logDir}/status.txt" />
     <plugin
         name="ant"
-        antscript="${env.BUILD_HOME}/releng.control/ant.sh"
-        buildfile="${env.BUILD_HOME}/releng.control/cc_project_build.xml"
+        antscript="${env.RELENG_CONTROL}/ant.sh"
+        buildfile="${env.RELENG_CONTROL}/cc_project_build.xml"
         target="Build"
         useQuiet="${env.USE_QUIET}"
         useLogger="${env.USE_LOGGER}"
         savelogdir="${logDir}"
         usedebug="${env.USE_DEBUG}"
-        antworkingdir="${env.BUILD_HOME}/releng.control" />
+        antworkingdir="${env.RELENG_CONTROL}" />
     <plugin
         name="project"
         buildafterfailed="true">
@@ -71,14 +71,14 @@
         </modificationset>
         <publishers>
             <antpublisher
-                antscript="${env.BUILD_HOME}/releng.control/ant.sh"
-                buildfile="${env.BUILD_HOME}/releng.control/cc_copyArtifacts.xml"
+                antscript="${env.RELENG_CONTROL}/ant.sh"
+                buildfile="${env.RELENG_CONTROL}/cc_copyArtifacts.xml"
                 target="copyArtifacts"
                 useQuiet="${env.USE_QUIET}"
                 useLogger="${env.USE_LOGGER}"
                 savelogdir="${logDir}"
                 usedebug="${env.USE_DEBUG}"
-                antworkingdir="${env.BUILD_HOME}/releng.control">
+                antworkingdir="${env.RELENG_CONTROL}">
                 <property
                     name="localArtifactsDirectory"
                     value="${env.BUILD_HOME}/artifacts/${project.name}" />
@@ -98,7 +98,7 @@
     </plugin>
     <plugin
         name="cvs"
-        cvsroot=":${env.WTP_CVS_PROTOCOL}:${env.WTP_CVS_USER}@${env.WTP_CVS_SERVER}:${env.WTP_CVS_ROOT}" />
+        cvsroot=":${env.BUILD_CVS_READ_PROTOCOL}:${env.BUILD_CVS_READ_USER}@${env.BUILD_CVS_SERVER}:${env.BUILD_CVS_ROOT}" />
 
     <!-- 
         
@@ -184,15 +184,18 @@
             <cvs
                 tag="R2_0_maintenance"
                 module="${env.RELENGMAPS}"
-                reallyQuiet="${env.CVS_REALLY_QUIET}" />
+                quiet="${env.CVS_QUIET}"
+                reallyquiet="${env.CVS_REALLY_QUIET}" />
             <cvs
                 tag="R1_0_maintenance"
                 module="${env.RELENGDALI}"
-                reallyQuiet="${env.CVS_REALLY_QUIET}" />
+                quiet="${env.CVS_QUIET}"
+                reallyquiet="${env.CVS_REALLY_QUIET}" />
             <cvs
                 tag="R1_0_maintenance"
                 module="${env.RELENGJSF}"
-                reallyQuiet="${env.CVS_REALLY_QUIET}" />
+                quiet="${env.CVS_QUIET}"
+                reallyquiet="${env.CVS_REALLY_QUIET}" />
         </modificationset>
         <schedule
             interval="${time.oneHour}">
@@ -231,15 +234,18 @@
         <cvs
         tag="R2_0_maintenance"
         module="${env.RELENGMAPS}"
-        reallyQuiet="${env.CVS_REALLY_QUIET}" />
+            quiet="${env.CVS_QUIET}"
+            reallyquiet="${env.CVS_REALLY_QUIET}" />
         <cvs
         tag="R1_0_maintenance"
         module="${env.RELENGDALI}"
-        reallyQuiet="${env.CVS_REALLY_QUIET}" />
+            quiet="${env.CVS_QUIET}"
+            reallyquiet="${env.CVS_REALLY_QUIET}" />
         <cvs
         tag="R1_0_maintenance"
         module="${env.RELENGJSF}"
-        reallyQuiet="${env.CVS_REALLY_QUIET}" />
+            quiet="${env.CVS_QUIET}"
+            reallyquiet="${env.CVS_REALLY_QUIET}"/>
         </modificationset>
         
         <schedule interval="${time.VeryVeryLongTime}">
@@ -277,13 +283,16 @@
             quietperiod="120">
             <cvs
                 module="${env.RELENGMAPS}"
-                reallyQuiet="${env.CVS_REALLY_QUIET}" />
+                quiet="${env.CVS_QUIET}"
+                reallyquiet="${env.CVS_REALLY_QUIET}" />
             <cvs
                 module="${env.RELENGJSF}"
-                reallyQuiet="${env.CVS_REALLY_QUIET}" />
+                quiet="${env.CVS_QUIET}"
+                reallyquiet="${env.CVS_REALLY_QUIET}" />
             <cvs
                 module="${env.RELENGDALI}"
-                reallyQuiet="${env.CVS_REALLY_QUIET}" />
+                quiet="${env.CVS_QUIET}"
+                reallyquiet="${env.CVS_REALLY_QUIET}" />
         </modificationset>
         <schedule
             interval="${time.VeryVeryLongTime}">
@@ -352,13 +361,16 @@
             quietperiod="120">
             <cvs
                 module="${env.RELENGMAPS}"
-                reallyQuiet="${env.CVS_REALLY_QUIET}" />
+            quiet="${env.CVS_QUIET}"
+            reallyquiet="${env.CVS_REALLY_QUIET}" />
             <cvs
                 module="${env.RELENGJSF}"
-                reallyQuiet="${env.CVS_REALLY_QUIET}" />
+            quiet="${env.CVS_QUIET}"
+            reallyquiet="${env.CVS_REALLY_QUIET}" />
             <cvs
                 module="${env.RELENGDALI}"
-                reallyQuiet="${env.CVS_REALLY_QUIET}" />
+            quiet="${env.CVS_QUIET}"
+            reallyquiet="${env.CVS_REALLY_QUIET}" />
         </modificationset>
         <schedule
             interval="${time.oneHour}">
@@ -404,13 +416,16 @@
         <modificationset quietperiod="120">
         <cvs
         module="${env.RELENGMAPS}"
-        reallyQuiet="${env.CVS_REALLY_QUIET}" />
+            quiet="${env.CVS_QUIET}"
+            reallyquiet="${env.CVS_REALLY_QUIET}"/>
         <cvs
         module="${env.RELENGJSF}"
-        reallyQuiet="${env.CVS_REALLY_QUIET}" />
+            quiet="${env.CVS_QUIET}"
+            reallyquiet="${env.CVS_REALLY_QUIET}" />
         <cvs
         module="${env.RELENGDALI}"
-        reallyQuiet="${env.CVS_REALLY_QUIET}" />
+            quiet="${env.CVS_QUIET}"
+            reallyquiet="${env.CVS_REALLY_QUIET}" />
         </modificationset>
         
         <schedule interval="${time.oneHour}">

releng.control/cc_project_build.xml

@@ -32,7 +32,7 @@
 	<target name="Build">
 
 		<!-- set main builder file name, based on distribution, buildBranch and build type -->
-		<property name="wtpBuilder" value="${env.BUILD_HOME}/projectBuilders/${checkoutprojectname}/${env.RELENGWTPBUILDER}/build.xml" />
+		<property name="wtpBuilder" value="${env.BUILD_HOME}/projectBuilders/${checkoutprojectname}/${env.RELENG_BUILDER}/build.xml" />
 
 		<antcall target="doCleanBuild" />
 
@@ -71,6 +71,7 @@
 		<ant antfile="${wtpBuilder}" target="upload" />
 	</target>
 
+    <!-- TODO: projects and tests should be variables, see build.properties in wtpBuilder -->
 	<target name="clean" depends="check.clean" if="doClean">
 		<delete dir="${env.BUILD_HOME}/projects/${checkoutprojectname}" failonerror="false" />
 		<delete dir="${env.BUILD_HOME}/tests/${checkoutprojectname}" failonerror="false" />

releng.control/cc_project_retest.xml

@@ -33,7 +33,7 @@
 
         <!-- set main builder file name, based on distribution, buildBranch and build type -->
         <property name="wtpBuilder"
-                  value="${env.BUILD_HOME}/projectBuilders/${checkoutprojectname}/${env.RELENGWTPBUILDER}/build.xml" />
+                  value="${env.BUILD_HOME}/projectBuilders/${checkoutprojectname}/${env.RELENG_BUILDER}/build.xml" />
 
        
 

releng.control/checkout.xml

@@ -31,16 +31,16 @@
         value="HEAD" />
     <property
         name="cvsProtocol"
-        value="${env.WTP_CVS_PROTOCOL}" />
+        value="${env.BUILD_CVS_WRITE_PROTOCOL}" />
     <property
         name="cvsUser"
-        value="${env.WTP_CVS_USER}" />
+        value="${env.BUILD_CVS_WRITE_USER}" />
     <property
         name="cvsServer"
-        value="${env.WTP_CVS_SERVER}" />
+        value="${env.BUILD_CVS_SERVER}" />
     <property
         name="cvsRoot"
-        value="${env.WTP_CVS_ROOT}" />
+        value="${env.BUILD_CVS_ROOT}" />
 
     <!-- end required defaults-if-not-set -->
 
@@ -279,15 +279,15 @@
             is fetched to control the rest of the build -->
         <property file="${codir}/${env.RELENG}/maps/build.cfg" />
         <delete
-            dir="${codir}/${env.RELENGWTPBUILDER}"
+            dir="${codir}/${env.RELENG_BUILDER}"
             failonerror="false" />
         <echo
-            message="Version tag for ${env.RELENGWTPBUILDER} is: ${wtpBuilderVersion}" />
+            message="Version tag for ${env.RELENG_BUILDER} is: ${wtpBuilderVersion}" />
         <cvs
             quiet="${env.CVS_QUIET}"
             reallyquiet="${env.CVS_REALLY_QUIET}"
             cvsRoot="${mapCvsRoot}"
-            package="${env.RELENGWTPBUILDER}"
+            package="${env.RELENG_BUILDER}"
             dest="${codir}"
             tag="${wtpBuilderVersion}" />
     </target>

releng.control/commonComputedVariables.shsource

@@ -8,7 +8,40 @@
 
 export PROJECT_HOME=${BUILD_HOME}/releng.control
 
-export PATH=${PATH}:${HOME}/bin:${ANT_HOME}/bin:${JAVA_HOME}/bin
+export ANT_WORKING=${BUILD_HOME}/antWorking
+export BASE_BUILDERS=${BUILD_HOME}/basebuilders
+export PROJECT_BUILDERS=${BUILD_HOME}/projectBuilders
+# 	Set this to the relative path for the 
+# 	directory that keeps the downloaded drivers 
+#   and pre-reqs
+export LOCAL_PREREQS_CACHE=${BUILD_HOME}/prereqsCache
+# must match cacheRoot in CC's web.xml 
+export CACHE_ROOT=${BUILD_HOME}/logcache
+# must correspond to what's specified in CC's web.xml file, and config
+export LOG_DIR=${BUILD_HOME}/logs
+export PROJECT_TESTS=${BUILD_HOME}/tests
+export PROJECT_PROJECTS=${BUILD_HOME}/projects
+export PROJECT_ARTIFACTS=${BUILD_HOME}/artifacts
+
+# in a fresh install, we need to make sure some basic directories exist
+mkdir -p ${ANT_WORKING}
+mkdir -p ${BASE_BUILDERS}
+mkdir -p ${PROJECT_BUILDERS}
+mkdir -p ${LOCAL_PREREQS_CACHE}
+mkdir -p ${CACHE_ROOT}
+mkdir -p ${LOG_DIR}
+mkdir -p ${PROJECT_TESTS}
+mkdir -p ${PROJECT_PROJECTS}
+mkdir -p ${PROJECT_ARTIFACTS}
+
+export RELENG_BUILDER=webtools.releng/releng.wtpbuilder
+
+export RELENG=webtools.maps/releng
+export RELENGMAPS=webtools.maps
+export RELENGJSF=webtools.maps/releng.jsf
+export RELENGDALI=webtools.maps/releng.dali
+
+export PATH=${PATH}:${ANT_HOME}/bin:${JAVA_HOME}/bin
 
 # project name is resolved dynamically, as CC runs
 export LOG_DIR=${BUILD_HOME}/logs
@@ -16,47 +49,47 @@ export LOG_DIR=${BUILD_HOME}/logs
 # need for some PPC or Linux issues 
 export JAVA_HIGH_ZIPFDS=500
 
-# Spec java 1.4 as default vm to use if
-# not otherwise specified. 
-# TODO: do we still need this?
-export JAVA_HOME=${JAVA_4_HOME}
+# for now, always use Java 5 as native "home", but 
+# scripts and paths may still use java 1.4
+export JAVA_HOME=${JAVA_5_HOME}
 
-# 	Set this to the relative path for the 
-# 	directory that keeps the downloaded drivers 
-#   and pre-reqs
-export LOCAL_PREREQS_CACHE=${BUILD_HOME}/downloads
+# for  builds on local machines, SKIPUSERS should be 'true' to avoid 
+# notifying users about those builds on your local machine
+# but if not defined thus far, set to false for production build machine.
+if [ -z ${SKIPUSERS} ] 
+then 
+	export SKIPUSERS=false;
+fi
 
 # We want everything in our processes to use UTC, even when we can't 
 # change the time or timezone on the build machine.
 export TZ='UTC-0';
 
 
-export RELENG=webtools.maps/releng
-export RELENGMAPS=webtools.maps
-export RELENGJSF=webtools.maps/releng.jsf
-export RELENGDALI=webtools.maps/releng.dali
-export RELENGWTPBUILDER=webtools.releng/releng.wtpbuilder
-
 export BUILD_INITIALIZED="true"
 
-echo " "
+echo 
 echo "    Project Home: ${PROJECT_HOME}"
 echo "    Build Home: ${BUILD_HOME}"
 echo "    DISPLAY: ${DISPLAY}"
-echo " "
+echo 
 echo "    JAVA_4_HOME: ${JAVA_4_HOME}"
 echo "        ${bootclasspath}"
-echo " "
+echo 
 echo "    JAVA_5_HOME: ${JAVA_5_HOME}"
 echo "        ${bootclasspath_15}"
-echo " "
-echo "     ANT_HOME: ${ANT_HOME}"
-echo " "
-echo "     PATH: ${PATH}"
-echo " "
-echo "      BASEOS: ${BASEOS}"
-echo "      BASEWS: ${BASEWS}"
+echo 
+echo "    ANT_HOME: ${ANT_HOME}"
+echo 
+echo "    PATH: ${PATH}"
+echo 
+echo "    BASEOS: ${BASEOS}"
+echo "    BASEWS: ${BASEWS}"
 echo "    BASEARCH: ${BASEARCH}"
-echo " "
-echo "   LOCAL_BUILD_PROPERTIES_DIR: ${LOCAL_BUILD_PROPERTIES_DIR}" 
-echo " "
+echo 
+echo "    LOCAL_BUILD_PROPERTIES_DIR: ${LOCAL_BUILD_PROPERTIES_DIR}" 
+echo 
+echo "    remember to check that X virtual frame buffer is running for headless JUnit testing";
+ps -ef | grep Xvfb
+echo
+

releng.control/commonVariations.shsource

 #!/bin/sh
 
+# This particular file is not used in production builds,
+# though there is a similar one (with same name) in main
+# directory that is.
+
 # If "local" users have this file on their path, 
 # that is, in "someDirectory" on the users local path, 
 # then the bash "include source" function will find that version
@@ -17,16 +21,17 @@
 
 export BUILD_HOME=/shared/webtools
 
-export DISPLAY=127.0.0.1:1.0
+# the following does not really vary, but must be specified "early" 
+export RELENG_CONTROL=${BUILD_HOME}/releng.control
 
-echo "remember to check X virtual frame buffer is running for headless JUnit testing";
-ps -ef | grep Xvfb
+export DISPLAY=127.0.0.1:1.0
 
 # these should be full, abosolute path to the respective home directories
 # on this particular build machine
 export ANT_HOME=/shared/webtools/apps/apache-ant-1.7.0
 export JAVA_4_HOME=/shared/webtools/apps/IBMJava2-ppc-142
 export JAVA_5_HOME=/shared/webtools/apps/ibm-java2-ppc-50
+
 export CCDIR=/shared/webtools/apps/cruisecontrol-bin-2.7.1
 
 export BASEOS=linux
@@ -39,22 +44,23 @@ export BASEARCH=ppc
 # See the 'localBuildExample' directory for a starting example
 export LOCAL_BUILD_PROPERTIES_DIR=
 
-export BUILD_BUILDMASTER_EMAIL=wtpBuild@eclipse.org
-export BUILD_RETURN_NAME=wtpBuild
+# note: buildmaster email address should be "real" address 
+# that is subscribed to the mailing list, if you want to send 
+# mail to mailing list. 
+export BUILD_BUILDMASTER_EMAIL=david_williams@us.ibm.com
+
+export BUILD_BUILDMASTER_RETURN_NAME=wtpBuild
 export BUILD_RESULT_URL=http://build.eclipse.org:7777/cruisecontrol/buildresults
 export BUILD_ALWAYS_TO_ADDRESS=david_williams@us.ibm.com
-
-# for  builds on local machines, SKIPUSERS should be 'true' to avoid 
-# notifying users about those builds on your local machine
-export SKIPUSERS=false
+export FAILED_ADDRESS=wtp-releng@eclipse.org
 
 # controls quietness of logging to console
 # false means more output to console, true uses ant's logger
-export USE_LOGGER=false
+export USE_LOGGER=true
 
 #contols ANT's quietness level
 # true logs error and warnings only 
-export USE_QUIET=false
+export USE_QUIET=true
 
 # controls ANT's debug level
 # setting to true will result in TONS of ouput
@@ -75,10 +81,10 @@ export USE_LOCAL_MAPS=false
 # TODO: if both of these are 'true' and error is generated by cvs ...
 # saying you can't set both to true.  
 # we could put in some logic to prevent that, someday. 
-export CVS_QUIET=false
-export CVS_REALLY_QUIET=false
+export CVS_QUIET=true
+export CVS_REALLY_QUIET=true
 #   these cvs values need to have write access if 
-#   build.trial is false, as on production machine. 
+#   tag.maps is true, as on production machine. 
 #   This is the ID used to both read, and 
 #   to tag releng map files.
 #
@@ -89,7 +95,14 @@ export CVS_REALLY_QUIET=false
 #   if protocol was pserver, but could be removed in that 
 #   case, if desired. 
 export CVS_RSH=ssh 
-export WTP_CVS_PROTOCOL=ext
-export WTP_CVS_USER=david_williams
-export WTP_CVS_SERVER=dev.eclipse.org
-export WTP_CVS_ROOT=/cvsroot/webtools
+export BUILD_CVS_READ_PROTOCOL=pserver
+export BUILD_CVS_READ_USER=anonymous
+export BUILD_CVS_WRITE_PROTOCOL=ext
+export BUILD_CVS_WRITE_USER=david_williams
+export BUILD_CVS_SERVER=dev.eclipse.org
+
+export BUILD_CVS_ROOT=/cvsroot/webtools
+
+# for  builds on local machines, SKIPUSERS should be 'true' to avoid 
+# notifying users about those builds on your local machine
+export SKIPUSERS=false

releng.control/getFromCVS-relengControl.sh

 #!/bin/sh
 
+# it is assumed we are executing this in RELENG_CONTROL or the parent of RELENG_CONTROL 
+RELENG_CONTROL=releng.control
+
 # This script file is to help get builds started "fresh", when 
-# the releng.control directory already exists on local file system.
-# While it is in the cvs repository in releng.control, it is 
+# the ${RELENG_CONTROL} directory already exists on local file system.
+# While it is in the cvs repository in ${RELENG_CONTROL}, it is 
 # meant to be executed from the parent directory
-# of releng.control on the file system. 
+# of ${RELENG_CONTROL} on the file system. 
 
 # export is used, instead of checkout, just to avoid the CVS directories and since this code
 # for a local build, there should never be a need to check it back in to CVS.
-#  
 
 
-if [ ! -e releng.control ] 
+if [ ! -e ${RELENG_CONTROL} ] 
 then
 	cd .. 
-	if [ ! -e releng.control ] 
+	if [ ! -e ${RELENG_CONTROL} ] 
 	then	
-    echo "releng.control does not exist as sub directory";
+    echo "${RELENG_CONTROL} does not exist as sub directory";
 	exit 1;
 	fi
 fi
 
-echo "	saving serialized state ..."
-cp releng.control/*.ser .
+echo
+echo "	saving serialized CC state ..."
+cp ${RELENG_CONTROL}/*.ser . >/dev/null 2>/dev/null
 
-echo "	removing all of releng.control ..."
-rm -fr releng.control/*
-rm -fr releng.control/.project
-mkdir -p releng.control
+echo "	removing all of ${RELENG_CONTROL} ..."
+rm -fr ${RELENG_CONTROL}/*
+rm -fr ${RELENG_CONTROL}/.project
+mkdir -p ${RELENG_CONTROL}
 
-echo "	checking out head of releng.control from cvs ..."
-cvs -d :pserver:anonymous@dev.eclipse.org:/cvsroot/webtools  export -d releng.control -r HEAD webtools.releng/releng.control
+echo "	checking out head of ${RELENG_CONTROL} from cvs ..."
+cvs -Q -f -d :pserver:anonymous@dev.eclipse.org:/cvsroot/webtools  export -d ${RELENG_CONTROL} -r HEAD webtools.releng/${RELENG_CONTROL}
 
-echo "	restoring serialized state ..."
-mv *.ser releng.control
+echo "    restore serialized CC state"
+mv *.ser ${RELENG_CONTROL} >/dev/null 2>/dev/null
 
-echo "	making sure releng.control files are executable and have proper EOL format ..."
-dos2unix -quiet -keepdate releng.control/*.sh* releng.control/**/*.properties releng.control/*.xml > /dev/null 2>/dev/null
-chmod +x releng.control/*.sh
+echo "    make sure releng control files are executable and have proper EOL"
+dos2unix ${RELENG_CONTROL}/*.sh* ${RELENG_CONTROL}/*.properties ${RELENG_CONTROL}/*.xml >/dev/null 2>>/dev/null
+chmod +x ${RELENG_CONTROL}/*.sh > /dev/null
+echo