Tags:
tag this topic
create new tag
view all tags
<!-- keep this as a security measure: #uncomment if the subject should only be modifiable by the listed groups * Set ALLOWTOPICCHANGE = Main.TWikiAdminGroup,Main.CMSAdminGroup * Set ALLOWTOPICRENAME = Main.TWikiAdminGroup,Main.CMSAdminGroup #uncomment this if you want the page only be viewable by the listed groups # * Set ALLOWTOPICVIEW = Main.TWikiAdminGroup,Main.CMSAdminGroup --> %TOC% %ICON{arrowleft}% Go to [[CMSTier3LogXX][previous page]] / [[CMSTier3LogXX][next page]] of Tier3 site log %M% ---+ 30. 07. 2012 Dealing with the fallout from the new memory limits Analyzing maximal memory usage of the last few days based on data recorded by ARCO: ---++ Comparing crab and non-crab jobs Histogram of maximal memory usage for CRAB based jobs: <verbatim> mysql> select count(*),round(maxvmem/1000000000,1) as maxmem_GB,name as example_name from view_accounting WHERE start_time >= '2012-07-20' AND name LIKE '%crab%' GROUP BY maxmem_GB ORDER BY maxmem_GB; +----------+-----------+--------------------------------------+ | count(*) | maxmem_GB | example_name | +----------+-----------+--------------------------------------+ | 1872 | 0.0 | naegelic_crab_0_120719_150727_3x2e6k | | 1 | 0.3 | naegelic_crab_0_120719_150727_3x2e6k | | 1 | 0.6 | naegelic_crab_0_120726_225537_b86az1 | | 2751 | 0.9 | naegelic_crab_0_120724_212404_7c56bu | | 493 | 1.0 | favaro_crab_0_120725_100244_y1b8m2 | | 1 | 1.1 | meier_f1_crab_0_120717_073931_54bu8y | | 1289 | 1.2 | meier_f1_crab_0_120717_073931_54bu8y | | 9 | 1.3 | meier_f1_crab_0_120720_080552_hs90g1 | | 2 | 2.2 | meier_f1_crab_0_120717_073931_54bu8y | | 152 | 3.2 | naegelic_crab_0_120725_210329_2zl6f7 | | 1 | 3.7 | naegelic_crab_0_120719_150727_3x2e6k | | 544 | 4.4 | naegelic_crab_0_120719_150727_3x2e6k | | 126 | 6.3 | naegelic_crab_0_120726_225537_b86az1 | +----------+-----------+--------------------------------------+ </verbatim> Same for non-crab jobs <verbatim> mysql> select count(*),round(maxvmem/1000000000,1) as maxmem_GB,name as example_name from view_accounting WHERE start_time >= '2012-07-20' AND name NOT LIKE '%crab%' GROUP BY maxmem_GB ORDER BY maxmem_GB; +----------+-----------+-----------------------------------------------+ | count(*) | maxmem_GB | example_name | +----------+-----------+-----------------------------------------------+ | 1732 | 0.0 | aCBAF | | 36 | 0.1 | QLOGIN | | 2805 | 0.2 | ssdl_9 | | 2425 | 0.3 | ssdl_6 | | 237 | 0.4 | ssdl_7 | | 148 | 0.5 | latino_step3_070_WZFull.root | | 98 | 0.6 | Gemschi | | 19 | 0.7 | MultiJet-Run2012A-PromptReco-v1-AOD | | 7 | 0.8 | upst198609 | | 10 | 0.9 | DoubleElectron-Run2012A-PromptReco-v1-AOD | | 76 | 1.0 | QCD_Pt-20_MuEnrichedPt-15_TuneZ2_7TeV_pythia6 | | 34 | 1.1 | job_scan.sh | | 890 | 1.2 | KV | | 35 | 1.3 | aCBAF | | 244 | 1.4 | aCBAF | | 1026 | 1.5 | brillante | | 207 | 1.6 | aCBAF | | 766 | 1.7 | aCBAF | | 923 | 1.8 | aCBAF | | 430 | 1.9 | aCBAF | | 242 | 2.0 | snoek_H300_A20_whatever_patlv2_v2_01bx3t | | 128 | 2.1 | bmm-v12-Run2012C-PromptReco-v1_2-0040.csh | | 51 | 2.2 | RMG40electron_pt_5_100_eta_0_1p5 | | 4 | 2.3 | bmm-v11-Run2011B-PromptReco-v1-0142.csh | | 50 | 2.6 | RMG17electron_pt_5_100_eta_0_1p5 | | 52 | 2.7 | snoek_H300_A20_whatever_patlv2_v2_01bx3t | | 68 | 2.8 | snoek_H300_A20_whatever_patlv2_v2_01bx3t | | 430 | 3.2 | Schubert | | 1 | 3.3 | cbmm-v12-Run2012C-PromptReco-v1_2-00.csh | | 876 | 6.3 | bmm-v11-Run2011A-PromptReco-v5-0150.csh | +----------+-----------+-----------------------------------------------+ </verbatim> ---++ Looking at the number of crab jobs below and above a 3GB threshold Crab jobs below the 3 GB threshold <verbatim> mysql> select count(*),round(maxvmem/1000000000,1) as maxmem_GB,username,name from view_accounting WHERE start_time >= '2012-07-01' AND maxvmem <= 3000000000 AND name LIKE '%crab%' GROUP BY maxmem_GB,username ORDER BY maxmem_GB; +----------+-----------+----------+-----------------------------------------------------------------------------------------------------------------------+ | count(*) | maxmem_GB | username | name | +----------+-----------+----------+-----------------------------------------------------------------------------------------------------------------------+ | 1087 | 0.0 | haweber | haweber_crab_PYTHIA6_SUSY_Stop400_to_bChi300_to_WLSP100-8TeV-GEN_FASTSIM-PU2012_Summer_inTimeOnly_s53nf7 | | 244 | 0.0 | leo | leo_crab_PYTHIA6_SUSY_LM10_sftsht_8TeV_cff_py_GEN_FASTSIM_HLT_PU-V3_wc80j4 | | 1000 | 0.0 | meier_f1 | meier_f1_crab_0_120717_073931_54bu8y | | 292 | 0.0 | mirena | mirena_crab_0_120703_180906_98xp6c | | 1162 | 0.0 | naegelic | naegelic_crab_0_120719_150727_3x2e6k | | 1 | 0.0 | peruzzi | qspymc_V02-06-12_multicrab.cfg | | 982 | 0.0 | snoek | snoek_crab_step2_20_8ps3j1 | | 1 | 0.1 | haweber | haweber_crab_PYTHIA6_SUSY_Stop400_to_bChi300_to_WLSP100-8TeV-GEN_FASTSIM-PU2012_Summer_inTimeOnly_cq721a | | 17 | 0.3 | naegelic | naegelic_crab_0_120719_140732_4km6e7 | | 3 | 0.4 | naegelic | naegelic_crab_0_120719_140732_4km6e7 | | 183 | 0.5 | snoek | snoek_crab_cpy_20_v389ih | | 93 | 0.6 | favaro | favaro_crab_0_120710_142546_32fjn0 | | 1 | 0.6 | naegelic | naegelic_crab_0_120726_225537_b86az1 | | 39 | 0.6 | snoek | snoek_crab_cpy_20_v389ih | | 367 | 0.7 | snoek | snoek_crab_cpy_20_6ohx37 | | 1 | 0.8 | peruzzi | qspyGJet_Pt_20_doubleEMEnriched_TuneZ2_7TeV_pythia6_Fall11_PU_S6_START42_V14B_v1_AODSIM_mc_V02-06-12_multicrab.cfg | | 7 | 0.9 | favaro | favaro_crab_0_120725_100244_y1b8m2 | | 29 | 0.9 | mirena | mirena_crab_0_120703_164846_0i2h4a | | 2746 | 0.9 | naegelic | naegelic_crab_0_120719_140732_4km6e7 | | 14 | 0.9 | snoek | snoek_crab_cpy_20_v389ih | | 1362 | 1.0 | favaro | favaro_crab_0_120705_124438_4jb8t7 | | 392 | 1.0 | haweber | haweber_crab_PYTHIA6_SUSY_Stop400_to_bChi300_to_WLSP100-8TeV-GEN_FASTSIM-PU2012_Summer_inTimeOnly_cq721a | | 541 | 1.0 | mirena | mirena_crab_0_120703_164846_0i2h4a | | 3 | 1.0 | naegelic | naegelic_crab_0_120719_140732_4km6e7 | | 1 | 1.1 | meier_f1 | meier_f1_crab_0_120717_073931_54bu8y | | 52 | 1.1 | mirena | mirena_crab_0_120703_164846_0i2h4a | | 2 | 1.1 | naegelic | naegelic_crab_0_120719_140732_4km6e7 | | 1440 | 1.2 | meier_f1 | meier_f1_crab_0_120717_073931_54bu8y | | 754 | 1.2 | mirena | mirena_crab_0_120703_164846_0i2h4a | | 3070 | 1.3 | haweber | haweber_crab_PYTHIA6_SUSY_Stop400_to_bChi300_to_WLSP100-8TeV-GEN_FASTSIM-PU2012_Summer_inTimeOnly_cq721a | | 11 | 1.3 | meier_f1 | meier_f1_crab_0_120717_073931_54bu8y | | 87 | 1.3 | mirena | mirena_crab_0_120703_164846_0i2h4a | | 142 | 1.4 | haweber | haweber_crab_PYTHIA6_SUSY_Stop400To_bChi300To_Sl200Nu_LSnu200_LSP100-8TeV-GEN_FASTSIM-PU2012_Summer_inTimeOnly_56y9ik | | 43 | 1.4 | mirena | mirena_crab_0_120703_164846_0i2h4a | | 66 | 1.5 | haweber | haweber_crab_PYTHIA6_SUSY_Stop400_to_bChi300_to_WLSP100-8TeV-GEN_FASTSIM-PU2012_Summer_inTimeOnly_cq721a | | 71 | 1.5 | mirena | mirena_crab_0_120703_164846_0i2h4a | | 103 | 1.6 | haweber | haweber_crab_PYTHIA6_SUSY_Stop400_to_bChi300_to_WLSP100-8TeV-GEN_FASTSIM-PU2012_Summer_inTimeOnly_cq721a | | 15 | 1.6 | mirena | mirena_crab_0_120705_095950_6df30j | | 1 | 1.6 | snoek | snoek_crab_step3_50_topup_u97ep4 | | 2 | 1.7 | mirena | mirena_crab_0_120705_095950_6df30j | | 1 | 1.8 | naegelic | naegelic_crab_0_120719_140732_4km6e7 | | 2 | 1.8 | snoek | snoek_crab_step3_50_14y2rg | | 3 | 1.9 | naegelic | naegelic_crab_0_120719_140732_4km6e7 | | 119 | 1.9 | snoek | snoek_crab_step3_80_394moh | | 7 | 2.0 | naegelic | naegelic_crab_0_120719_140732_4km6e7 | | 82 | 2.0 | snoek | snoek_crab_step3_80_394moh | | 2 | 2.1 | naegelic | naegelic_crab_0_120719_140732_4km6e7 | | 9 | 2.1 | snoek | snoek_crab_step3_20_sd1v78 | | 3 | 2.2 | meier_f1 | meier_f1_crab_0_120717_073931_54bu8y | | 19 | 2.2 | snoek | snoek_crab_step3_50_14y2rg | | 16 | 2.3 | snoek | snoek_crab_step3_50_14y2rg | | 31 | 2.4 | snoek | snoek_crab_step3_50_14y2rg | | 3 | 2.5 | snoek | snoek_crab_step3_50_14y2rg | | 4 | 2.6 | snoek | snoek_crab_step2_v2_5mz7q0 | | 118 | 2.7 | snoek | snoek_crab_step2_v2_5mz7q0 | | 398 | 2.8 | snoek | snoek_crab_step2_20_8ps3j1 | | 117 | 2.9 | leo | leo_crab_PYTHIA6_SUSY_LM7_sftsht_8TeV_cff_py_GEN_FASTSIM_HLT_PU-V3_4r90jh | | 281 | 2.9 | snoek | snoek_crab_step2_20_8ps3j1 | | 316 | 3.0 | leo | leo_crab_PYTHIA6_SUSY_LM7_sftsht_8TeV_cff_py_GEN_FASTSIM_HLT_PU-V3_4r90jh | | 3 | 3.0 | snoek | snoek_crab_step2_20_8ps3j1 | +----------+-----------+----------+-----------------------------------------------------------------------------------------------------------------------+ 60 rows in set (2.30 sec) </verbatim> Crab jobs above the 3 GB threshold <verbatim> mysql> select count(*),round(maxvmem/1000000000,1) as maxmem_GB,username,name from view_accounting WHERE start_time >= '2012-07-01' AND maxvmem > 3000000000 AND name LIKE '%crab%' GROUP BY maxmem_GB,username ORDER BY maxmem_GB; +----------+-----------+----------+---------------------------------------------------------------------------+ | count(*) | maxmem_GB | username | name | +----------+-----------+----------+---------------------------------------------------------------------------+ | 44 | 3.0 | leo | leo_crab_PYTHIA6_SUSY_LM7_sftsht_8TeV_cff_py_GEN_FASTSIM_HLT_PU-V3_4r90jh | | 1 | 3.1 | leo | leo_crab_PYTHIA6_SUSY_LM7_sftsht_8TeV_cff_py_GEN_FASTSIM_HLT_PU-V3_4r90jh | | 152 | 3.2 | naegelic | naegelic_crab_0_120725_210329_2zl6f7 | | 1 | 3.2 | snoek | snoek_crab_step2_20_second_58p3tb | | 2 | 3.3 | snoek | snoek_crab_step2_20_second_58p3tb | | 1 | 3.7 | naegelic | naegelic_crab_0_120719_150727_3x2e6k | | 1 | 4.0 | snoek | snoek_crab_step3_50_14y2rg | | 616 | 4.4 | naegelic | naegelic_crab_0_120719_150727_3x2e6k | | 126 | 6.3 | naegelic | naegelic_crab_0_120726_225537_b86az1 | +----------+-----------+----------+---------------------------------------------------------------------------+ 9 rows in set (1.79 sec) </verbatim> -- Main.DerekFeichtinger - 2012-07-30 ---------------- %ICON{arrowleft}% Go to [[CMSTier3LogXX][previous page]] / [[CMSTier3LogXX][next page]] of Tier3 site log %M%
E
dit
|
A
ttach
|
Watch
|
P
rint version
|
H
istory
: r1
|
B
acklinks
|
V
iew topic
|
Ra
w
edit
|
M
ore topic actions
Topic revision: r1 - 2012-07-30
-
DerekFeichtinger
CmsTier3
Log In
CmsTier3 Web
Create New Topic
Index
Search
Changes
Notifications
Statistics
Preferences
User Pages
Main Page
Policies
Monitoring Storage Space
Monitoring Slurm Usage
Physics Groups
Steering Board Meetings
Admin Pages
AdminArea
Cluster Specs
Home
Site map
CmsTier3 web
LCGTier2 web
PhaseC web
Main web
Sandbox web
TWiki web
CmsTier3 Web
Create New Topic
Index
Search
Changes
Notifications
RSS Feed
Statistics
Preferences
P
View
Raw View
Print version
Find backlinks
History
More topic actions
Edit
Raw edit
Attach file or image
Edit topic preference settings
Set new parent
More topic actions
Account
Log In
E
dit
A
ttach
Copyright © 2008-2024 by the contributing authors. All material on this collaboration platform is the property of the contributing authors.
Ideas, requests, problems regarding TWiki?
Send feedback