Skip to content

Commit 10330ec

Browse files
fstagnifstagni
authored
Merge pull request #272 from fstagni/newMaster
New master
2 parents ff53464 + 7d1766b commit 10330ec

6 files changed

Lines changed: 204 additions & 277 deletions

File tree

‎.github/workflows/integration.yml‎

Lines changed: 37 additions & 120 deletions
Large diffs are not rendered by default.

‎Pilot/pilotCommands.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -508,6 +508,8 @@ def _getBasicsCFG(self):
508508
self.cfg.append('-S "%s"' % self.pp.setup)
509509
if self.pp.configServer:
510510
self.cfg.append('-C "%s"' % self.pp.configServer)
511+
if self.pp.preferredURLPatterns:
512+
self.cfg.append("-o /DIRAC/PreferredURLPatterns=%s" % quote(",".join(self.pp.preferredURLPatterns)))
511513
if self.pp.releaseProject:
512514
self.cfg.append('-e "%s"' % self.pp.releaseProject)
513515
self.cfg.append("-o /LocalSite/ReleaseProject=%s" % self.pp.releaseProject)
@@ -834,7 +836,7 @@ def execute(self):
834836
archScript = self.pp.architectureScript
835837
if self.pp.architectureScript.split(" ")[0] == "dirac-apptainer-exec":
836838
archScript = " ".join(self.pp.architectureScript.split(" ")[1:])
837-
839+
838840
architectureCmd = "%s %s -ddd" % (archScript, " ".join(cfg))
839841

840842
if self.pp.architectureScript.split(" ")[0] == "dirac-apptainer-exec":
@@ -872,10 +874,12 @@ def execute(self):
872874

873875
return localArchitecture
874876

877+
875878
class ConfigureArchitectureWithoutCLI(CommandBase):
876879
"""This command determines the platform.
877880
Separated from the ConfigureDIRAC command for easier extensibility.
878881
"""
882+
879883
def getPlatformString(self):
880884
# Modified to return our desired platform string, R. Graciani
881885
platformTuple = (platform.system(), platform.machine())
@@ -903,7 +907,6 @@ def execute(self):
903907
self.log.error("Configuration error [ERROR %s]" % str(e))
904908
self.exitWithError(1)
905909

906-
907910
cfg = ["-FDMH"] # force update, skip CA checks, skip CA download, skip VOMS
908911
if self.pp.useServerCertificate:
909912
cfg.append("--UseServerCertificate")

‎Pilot/pilotTools.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -793,7 +793,7 @@ def executeAndGetOutput(self, cmd, environDict=None):
793793
outChunk = outChunk.decode("utf-8")
794794
# Strip unicode replacement characters
795795
# Ensure correct type conversion in Python 2
796-
outChunk = str(outChunk.replace(u"\ufffd", ""))
796+
outChunk = str(outChunk.replace("\ufffd", ""))
797797
# Avoid potential str() issues in Py2
798798
outChunk = unicode(outChunk) # pylint: disable=undefined-variable
799799
else:
@@ -908,6 +908,7 @@ def __init__(self):
908908
self.site = ""
909909
self.setup = ""
910910
self.configServer = ""
911+
self.preferredURLPatterns = ""
911912
self.ceName = ""
912913
self.ceType = ""
913914
self.queueName = ""
@@ -1306,6 +1307,8 @@ def __initJSON2(self):
13061307
# the generic one (a list):
13071308
self.configServer = ",".join([str(pv).strip() for pv in self.pilotJSON["ConfigurationServers"]])
13081309

1310+
self.preferredURLPatterns = self.pilotJSON.get("PreferredURLPatterns", self.preferredURLPatterns)
1311+
13091312
# version(a comma separated values in a string). We take the first one. (the default value defined in the code)
13101313
dVersion = pilotOptions.get("Version", self.releaseVersion)
13111314
if dVersion:

‎Pilot/tests/Test_simplePilotLogger.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,5 +172,6 @@ def test_executeAndGetOutput(self, popenMock, argvmock):
172172
self.stdout_mock.truncate()
173173
self.stderr_mock.truncate()
174174

175+
175176
if __name__ == "__main__":
176177
unittest.main()

‎tests/CI/pilot_newSchema.json‎

Lines changed: 155 additions & 152 deletions
Original file line numberDiff line numberDiff line change
@@ -1,157 +1,160 @@
11
{
2-
"timestamp": "2023-02-13T14:34:26.725499",
3-
"CEs": {
4-
"jenkins.cern.ch": {
5-
"Site": "VAR_JENKINS_SITE",
6-
"GridCEType": "TEST"
7-
},
8-
"jenkins-full.cern.ch": {
9-
"Site": "VAR_JENKINS_SITE",
10-
"GridCEType": "TEST-FULL"
11-
},
12-
"jenkins-mp.cern.ch": {
13-
"Site": "VAR_JENKINS_SITE",
14-
"GridCEType": "TEST-MP"
15-
},
16-
"jenkins-mp-full.cern.ch": {
17-
"Site": "VAR_JENKINS_SITE",
18-
"GridCEType": "TEST-FULL"
19-
},
20-
"jenkins-singularity.cern.ch": {
21-
"Site": "VAR_JENKINS_SITE",
22-
"GridCEType": "TEST",
23-
"LocalCEType": "Singularity",
24-
"InstallDIRACInContainer": "True"
25-
},
26-
"jenkins-singularity-full.cern.ch": {
27-
"Site": "VAR_JENKINS_SITE",
28-
"GridCEType": "TEST-FULL",
29-
"LocalCEType": "Singularity"
30-
},
31-
"jenkins-mp-pool.cern.ch": {
32-
"Site": "VAR_JENKINS_SITE",
33-
"Queue": "VAR_JENKINS_QUEUE",
34-
"GridCEType": "TEST-MP",
35-
"LocalCEType": "Pool"
36-
},
37-
"jenkins-mp-pool-full.cern.ch": {
38-
"Site": "VAR_JENKINS_SITE",
39-
"GridCEType": "TEST-FULL",
40-
"LocalCEType": "InProcess",
41-
"VAR_JENKINS_QUEUE": {
42-
"LocalCEType": "Pool"
43-
}
44-
},
45-
"jenkins-mp-pool-sudo.cern.ch": {
46-
"Site": "VAR_JENKINS_SITE",
47-
"GridCEType": "TEST-MP",
48-
"VAR_JENKINS_QUEUE": {
49-
"LocalCEType": "Pool/Sudo"
50-
}
51-
},
52-
"jenkins-mp-pool-sudo-full.cern.ch": {
53-
"Site": "VAR_JENKINS_SITE",
54-
"GridCEType": "TEST-FULL",
55-
"VAR_JENKINS_QUEUE": {
56-
"LocalCEType": "Pool/Sudo"
57-
}
58-
},
59-
"jenkins-mp-pool-singularity.cern.ch": {
60-
"Site": "VAR_JENKINS_SITE",
61-
"GridCEType": "TEST-MP",
62-
"VAR_JENKINS_QUEUE": {
63-
"LocalCEType": "Pool/Singularity"
64-
}
65-
},
66-
"jenkins-mp-pool-singularity-full.cern.ch": {
67-
"Site": "VAR_JENKINS_SITE",
68-
"GridCEType": "TEST-FULL",
69-
"VAR_JENKINS_QUEUE": {
70-
"LocalCEType": "Pool/Singularity"
71-
}
72-
},
73-
"jenkins-lhcb.cern.ch": {
74-
"Site": "VAR_JENKINS_SITE",
75-
"GridCEType": "LHCb"
76-
},
77-
"jenkins-lhcb-d.cern.ch": {
78-
"Site": "VAR_JENKINS_SITE",
79-
"GridCEType": "LHCbD"
80-
},
81-
"jenkins-lhcb-dd.cern.ch": {
82-
"Site": "VAR_JENKINS_SITE",
83-
"GridCEType": "LHCbDD"
84-
},
2+
"timestamp": "2023-02-13T14:34:26.725499",
3+
"CEs": {
4+
"jenkins.cern.ch": {
5+
"Site": "VAR_JENKINS_SITE",
6+
"GridCEType": "TEST"
7+
},
8+
"jenkins-full.cern.ch": {
9+
"Site": "VAR_JENKINS_SITE",
10+
"GridCEType": "TEST-FULL"
11+
},
12+
"jenkins-mp.cern.ch": {
13+
"Site": "VAR_JENKINS_SITE",
14+
"GridCEType": "TEST-MP"
15+
},
16+
"jenkins-mp-full.cern.ch": {
17+
"Site": "VAR_JENKINS_SITE",
18+
"GridCEType": "TEST-FULL"
19+
},
20+
"jenkins-singularity.cern.ch": {
21+
"Site": "VAR_JENKINS_SITE",
22+
"GridCEType": "TEST",
23+
"LocalCEType": "Singularity",
24+
"InstallDIRACInContainer": "True"
25+
},
26+
"jenkins-singularity-full.cern.ch": {
27+
"Site": "VAR_JENKINS_SITE",
28+
"GridCEType": "TEST-FULL",
29+
"LocalCEType": "Singularity"
30+
},
31+
"jenkins-mp-pool.cern.ch": {
32+
"Site": "VAR_JENKINS_SITE",
33+
"Queue": "VAR_JENKINS_QUEUE",
34+
"GridCEType": "TEST-MP",
35+
"LocalCEType": "Pool"
36+
},
37+
"jenkins-mp-pool-full.cern.ch": {
38+
"Site": "VAR_JENKINS_SITE",
39+
"GridCEType": "TEST-FULL",
40+
"LocalCEType": "InProcess",
41+
"VAR_JENKINS_QUEUE": {
42+
"LocalCEType": "Pool"
43+
}
44+
},
45+
"jenkins-mp-pool-sudo.cern.ch": {
46+
"Site": "VAR_JENKINS_SITE",
47+
"GridCEType": "TEST-MP",
48+
"VAR_JENKINS_QUEUE": {
49+
"LocalCEType": "Pool/Sudo"
50+
}
51+
},
52+
"jenkins-mp-pool-sudo-full.cern.ch": {
53+
"Site": "VAR_JENKINS_SITE",
54+
"GridCEType": "TEST-FULL",
55+
"VAR_JENKINS_QUEUE": {
56+
"LocalCEType": "Pool/Sudo"
57+
}
58+
},
59+
"jenkins-mp-pool-singularity.cern.ch": {
60+
"Site": "VAR_JENKINS_SITE",
61+
"GridCEType": "TEST-MP",
62+
"VAR_JENKINS_QUEUE": {
63+
"LocalCEType": "Pool/Singularity"
64+
}
65+
},
66+
"jenkins-mp-pool-singularity-full.cern.ch": {
67+
"Site": "VAR_JENKINS_SITE",
68+
"GridCEType": "TEST-FULL",
69+
"VAR_JENKINS_QUEUE": {
70+
"LocalCEType": "Pool/Singularity"
71+
}
72+
},
73+
"jenkins-lhcb.cern.ch": {
74+
"Site": "VAR_JENKINS_SITE",
75+
"GridCEType": "LHCb"
76+
},
77+
"jenkins-lhcb-d.cern.ch": {
78+
"Site": "VAR_JENKINS_SITE",
79+
"GridCEType": "LHCbD"
80+
},
81+
"jenkins-lhcb-dd.cern.ch": {
82+
"Site": "VAR_JENKINS_SITE",
83+
"GridCEType": "LHCbDD"
84+
},
8585
"jenkins-revised.cern.ch": {
86-
"Site": "VAR_JENKINS_SITE",
87-
"GridCEType": "TEST-REVISED-COMMANDS"
86+
"Site": "VAR_JENKINS_SITE",
87+
"GridCEType": "TEST-REVISED-COMMANDS"
8888
}
89-
},
90-
"Defaults": {
91-
"Pilot": {
92-
"RemoteLogging": "False",
93-
"Commands": {
94-
"TEST": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements",
95-
"TEST-MP": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements",
96-
"TEST-FULL": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements, LaunchAgent",
97-
"LHCb": "CheckWorkerNode, LHCbInstallDIRAC, LHCbConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements",
98-
"LHCbD": "CheckWorkerNode, InstallDIRAC, LHCbConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements",
99-
"LHCbDD": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements",
100-
"TEST-REVISED-COMMANDS": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitectureWithoutCLI, ConfigureCPURequirements"
89+
},
90+
"Defaults": {
91+
"Pilot": {
92+
"RemoteLogging": "False",
93+
"Commands": {
94+
"TEST": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements",
95+
"TEST-MP": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements",
96+
"TEST-FULL": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements, LaunchAgent",
97+
"LHCb": "CheckWorkerNode, LHCbInstallDIRAC, LHCbConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements",
98+
"LHCbD": "CheckWorkerNode, InstallDIRAC, LHCbConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements",
99+
"LHCbDD": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, LHCbAddCVMFSTags, CheckWNCapabilities, ConfigureSite, ConfigureArchitecture, ConfigureCPURequirements",
100+
"TEST-REVISED-COMMANDS": "CheckWorkerNode, InstallDIRAC, ConfigureBasics, RegisterPilot, CheckCECapabilities, CheckWNCapabilities, ConfigureSite, ConfigureArchitectureWithoutCLI, ConfigureCPURequirements"
101101
}
102-
}
103-
},
104-
"gridpp": {
105-
"Pilot": {
106-
"Version": "VAR_DIRAC_VERSION",
107-
"Extensions": "None",
108-
"CheckVersion": "False",
109-
"pilotFileServer": "lbcertifdirac70.cern.ch:8443",
110-
"pilotRepoBranch": "does_not_matter",
111-
"pilotRepo": "https://github.com/does_not_matter/Pilot.git",
112-
"GenericPilotGroup": "gridpp_pilot",
113-
"GenericPilotDN": "VAR_USERDN_GRIDPP",
114-
"RemoteLogging": "False",
115-
"RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging",
116-
"UploadSE": "UKI-LT2-IC-HEP-disk",
117-
"UploadPath": "/gridpp/pilotlogs/",
118-
"RemoteLoggerTimerInterval": 0,
119-
"LoggingShifterName": "GridPPLogManager",
120-
"PilotLogLevel": "DEBUG",
121-
"CVMFS_locations": "/cvmfs/grid.cern.ch, /cvmfs/dirac.egi.eu"
122-
}
123-
},
124-
"dteam": {
125-
"Pilot": {
126-
"Version": "VAR_DIRAC_VERSION",
127-
"CheckVersion": "False",
128-
"pilotFileServer": "lbcertifdirac70.cern.ch:8443",
129-
"pilotRepoBranch": "should_not_matter",
130-
"pilotRepo": "https://github.com/should_not_matter/Pilot.git",
131-
"GenericPilotGroup": "dteam_pilot",
132-
"GenericPilotDN": "VAR_USERDN",
133-
"RemoteLogging": "True",
134-
"RemoteLoggerCEsWhiteList": "jenkins.cern.ch, jenkins-mp-pool.cern.ch",
135-
"RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging",
136-
"RemoteLoggerTimerInterval": 0,
137-
"PilotLogLevel": "DEBUG",
138-
"CVMFS_locations": "/cvmfs/grid.cern.ch, /cvmfs/dirac.egi.eu"
139-
}
140-
},
141-
"LHCb": {
142-
"Pilot": {
143-
"Version": "VAR_DIRAC_VERSION",
144-
"CheckVersion": "False",
145-
"pilotFileServer": "lbcertifdirac70.cern.ch:8443",
146-
"GenericPilotGroup": "dteam_pilot",
147-
"GenericPilotDN": "VAR_USERDN",
148-
"RemoteLogging": "False",
149-
"RemoteLoggerTimerInterval": 0,
150-
"PilotLogLevel": "DEBUG",
151-
"CVMFS_locations": "/cvmfs/lhcb.cern.ch/, $VO_LHCB_SW_DIR"
152-
}
153-
},
154-
"ConfigurationServers": [
155-
"VAR_CS"
156-
]
102+
}
103+
},
104+
"gridpp": {
105+
"Pilot": {
106+
"Version": "VAR_DIRAC_VERSION",
107+
"Extensions": "None",
108+
"CheckVersion": "False",
109+
"pilotFileServer": "lbcertifdirac70.cern.ch:8443",
110+
"pilotRepoBranch": "does_not_matter",
111+
"pilotRepo": "https://github.com/does_not_matter/Pilot.git",
112+
"GenericPilotGroup": "gridpp_pilot",
113+
"GenericPilotDN": "VAR_USERDN_GRIDPP",
114+
"RemoteLogging": "False",
115+
"RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging",
116+
"UploadSE": "UKI-LT2-IC-HEP-disk",
117+
"UploadPath": "/gridpp/pilotlogs/",
118+
"RemoteLoggerTimerInterval": 0,
119+
"LoggingShifterName": "GridPPLogManager",
120+
"PilotLogLevel": "DEBUG",
121+
"CVMFS_locations": "/cvmfs/grid.cern.ch, /cvmfs/dirac.egi.eu"
122+
}
123+
},
124+
"dteam": {
125+
"Pilot": {
126+
"Version": "VAR_DIRAC_VERSION",
127+
"CheckVersion": "False",
128+
"pilotFileServer": "lbcertifdirac70.cern.ch:8443",
129+
"pilotRepoBranch": "should_not_matter",
130+
"pilotRepo": "https://github.com/should_not_matter/Pilot.git",
131+
"GenericPilotGroup": "dteam_pilot",
132+
"GenericPilotDN": "VAR_USERDN",
133+
"RemoteLogging": "True",
134+
"RemoteLoggerCEsWhiteList": "jenkins.cern.ch, jenkins-mp-pool.cern.ch",
135+
"RemoteLoggerURL": "https://lbcertifdirac70.cern.ch:8443/WorkloadManagement/TornadoPilotLogging",
136+
"RemoteLoggerTimerInterval": 0,
137+
"PilotLogLevel": "DEBUG",
138+
"CVMFS_locations": "/cvmfs/grid.cern.ch, /cvmfs/dirac.egi.eu"
139+
}
140+
},
141+
"LHCb": {
142+
"Pilot": {
143+
"Version": "VAR_DIRAC_VERSION",
144+
"CheckVersion": "False",
145+
"pilotFileServer": "lbcertifdirac70.cern.ch:8443",
146+
"GenericPilotGroup": "dteam_pilot",
147+
"GenericPilotDN": "VAR_USERDN",
148+
"RemoteLogging": "False",
149+
"RemoteLoggerTimerInterval": 0,
150+
"PilotLogLevel": "DEBUG",
151+
"CVMFS_locations": "/cvmfs/lhcb.cern.ch/, $VO_LHCB_SW_DIR"
152+
}
153+
},
154+
"ConfigurationServers": [
155+
"VAR_CS"
156+
],
157+
"PreferredURLPatterns": [
158+
"VAR_PREF_URL_PATTERN"
159+
]
157160
}

‎tests/CI/pilot_oldSchema.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,8 +75,8 @@
7575
]
7676
},
7777
"Logging": {
78-
"LoggingType":"LOCAL_FILE",
79-
"LocalOutputFile":"myFile"
78+
"LoggingType": "LOCAL_FILE",
79+
"LocalOutputFile": "myFile"
8080
},
8181
"NagiosProbes": "WN-mjf, WN-sft-csh, WN-sft-voms, WN-sft-vo-swdir",
8282
"Version": "VAR_DIRAC_VERSION",

0 commit comments

Comments
 (0)