<!-- keep this as a security measure: #uncomment if the subject should only be modifiable by the listed groups * Set ALLOWTOPICCHANGE = Main.TWikiAdminGroup,Main.CMSAdminGroup * Set ALLOWTOPICRENAME = Main.TWikiAdminGroup,Main.CMSAdminGroup #uncomment this if you want the page only be viewable by the listed groups # * Set ALLOWTOPICVIEW = Main.TWikiAdminGroup,Main.CMSAdminGroup --> *CMS Tier-3 Upgrade Planning Page* <!-- # Use the attached form to define title and summary of this news item. The details you can fill in directly on this wiki page. --> ---+ %FORMFIELD{"Title"}% ---++ Summary %FORMFIELD{"Summary"}% ---++ Details ---+++ Planned / Status | *Planned* | *Status* | | UIs UMD3 upgrade | %Y% | | UIs latest dCache SRM RPM | %Y% | | UIs latest [[http://italiangrid.github.io/voms/2013/10/25/voms-clients-3-0-4-released.html][voms clients]] | %Y% | | t3bdii UMD3 upgrade | %Y% | | t3se01 bdii UMD3 upgrade | %Y% | | t3se01 PG 9 upgrade | %Y% | | t3dcachedb03 PG 9 upgrade | %Y% | | t3se01 2.6 upgrade | %Y% | | t3dcachedb03 2.6 upgrade | %Y% | | t3ldap01 New Primary Groups | %Y% | | t3dcachedb03 gPlazma2 aware of the new Primary Groups | %Y% | | t3dcachedb03 huge =chgrp -R= vs =/pnfs/psi.ch/cms/trivcat/store/user= | %Y% | | Solaris pools upgrade to 2.6 + JDK 1.7 | %Y% | | Linux pools upgrade to 2.6 + JDK 1.7 | %Y% but removed the xrootd monitor plugin | | %RED%Daniel discovered that the dCache xrootd plugin on fs13,14 causes issues%ENDCOLOR% | Removed RPM on fs13,14 | Error evidence: <pre> [root@t3fs14 meta]# cat /mnt/data10/t3fs14_cms/pool/meta/je.info.0 131108 16:52:45:165 INFO [/mnt/data10/t3fs14_cms/pool/meta] Chose lowest utilized file for cleaning. fileChosen: 0xa totalUtilization: 37 bestFileUtilization: 8 lnSizeCorrectionFactor: NaN isProbe: false 131108 16:52:48:530 INFO [/mnt/data10/t3fs14_cms/pool/meta] CleanerRun 1 ends on file 0xa probe=false invokedFromDaemon=true finished=true fileDeleted=false nEntriesRead=47022 nINsObsolete=7 nINsCleaned=703 nINsDead=558 nINsMigrated=145 nBINDeltasObsolete=0 nBINDeltasCleaned=0 nBINDeltasDead=0 nBINDeltasMigrated=0 nLNsObsolete=13897 nLNsCleaned=32414 nLNsDead=21441 nLNsMigrated=10973 nLNsMarked=0 nLNQueueHits=5292 nLNsLocked=0 logSummary=<CleanerLogSummary endFileNumAtLastAdjustment="0xe" initialAdjustments="1" recentLNSizesAndCounts=""> inSummary=<INSummary totalINCount="710" totalINSize="2092724" totalBINDeltaCount="0" totalBINDeltaSize="0" obsoleteINCount="7" obsoleteINSize="13943" obsoleteBINDeltaCount="0" obsoleteBINDeltaSize="0"/> estFileSummary=<summary totalCount="86816" totalSize="9997894" totalINCount="710" totalINSize="2092724" totalLNCount="46311" totalLNSize="6631427" maxLNSize="0" obsoleteINCount="1062" obsoleteLNCount="42016" obsoleteLNSize="1724490" obsoleteLNSizeCounted="20211" getObsoleteSize="9190411" getObsoleteINSize="2092724" getObsoleteLNSize="5823944" getMaxObsoleteSize="9190411" getMaxObsoleteLNSize="5823944" getAvgObsoleteLNSizeNotCounted="188.00525"/> recalcFileSummary=<summary totalCount="86816" totalSize="9997894" totalINCount="710" totalINSize="2092724" totalLNCount="46311" totalLNSize="6631427" maxLNSize="0" obsoleteINCount="7" obsoleteLNCount="13897" obsoleteLNSize="1040847" obsoleteLNSizeCounted="13897" getObsoleteSize="2335222" getObsoleteINSize="20632" getObsoleteLNSize="1040847" getMaxObsoleteSize="2335222" getMaxObsoleteLNSize="1040847" getAvgObsoleteLNSizeNotCounted="NaN"/> lnSizeCorrection=NaN newLnSizeCorrection=NaN estimatedUtilization=8 correctedUtilization=8 recalcUtilization=77 131108 16:52:48:530 INFO [/mnt/data10/t3fs14_cms/pool/meta] Chose lowest utilized file for cleaning. fileChosen: 0xc totalUtilization: 40 bestFileUtilization: 13 lnSizeCorrectionFactor: NaN isProbe: false 131108 16:52:48:814 INFO [/mnt/data10/t3fs14_cms/pool/meta] CleanerRun 2 ends on file 0xc probe=false invokedFromDaemon=true finished=true fileDeleted=false nEntriesRead=42335 nINsObsolete=72 nINsCleaned=913 nINsDead=775 nINsMigrated=138 nBINDeltasObsolete=0 nBINDeltasCleaned=0 nBINDeltasDead=0 nBINDeltasMigrated=0 nLNsObsolete=31517 nLNsCleaned=9831 nLNsDead=5833 nLNsMigrated=3998 nLNsMarked=0 nLNQueueHits=209 nLNsLocked=0 logSummary=<CleanerLogSummary endFileNumAtLastAdjustment="0xe" initialAdjustments="2" recentLNSizesAndCounts="Cor:185141/6694-Est:2126328/10692 "> inSummary=<INSummary totalINCount="985" totalINSize="3572196" totalBINDeltaCount="0" totalBINDeltaSize="0" obsoleteINCount="72" obsoleteINSize="171771" obsoleteBINDeltaCount="0" obsoleteBINDeltaSize="0"/> estFileSummary=<summary totalCount="83451" totalSize="9999934" totalINCount="985" totalINSize="3572196" totalLNCount="41348" totalLNSize="5110470" maxLNSize="0" obsoleteINCount="847" obsoleteLNCount="37350" obsoleteLNSize="2984142" obsoleteLNSizeCounted="30656" getObsoleteSize="8704378" getObsoleteINSize="3071726" getObsoleteLNSize="4315384" getMaxObsoleteSize="8704378" getMaxObsoleteLNSize="4315384" getAvgObsoleteLNSizeNotCounted="198.87093"/> recalcFileSummary=<summary totalCount="83451" totalSize="9999934" totalINCount="985" totalINSize="3572196" totalLNCount="41348" totalLNSize="5110470" maxLNSize="0" obsoleteINCount="72" obsoleteLNCount="31517" obsoleteLNSize="3169283" obsoleteLNSizeCounted="31517" getObsoleteSize="4747665" getObsoleteINSize="261114" getObsoleteLNSize="3169283" getMaxObsoleteSize="4747665" getMaxObsoleteLNSize="3169283" getAvgObsoleteLNSizeNotCounted="NaN"/> lnSizeCorrection=NaN newLnSizeCorrection=0.1390739 estimatedUtilization=13 correctedUtilization=13 recalcUtilization=53 131108 16:52:48:815 INFO [/mnt/data10/t3fs14_cms/pool/meta] Chose lowest utilized file for cleaning. fileChosen: 0x6 totalUtilization: 48 bestFileUtilization: 29 lnSizeCorrectionFactor: 0.1390739 isProbe: false 131108 16:52:49:698 INFO [/mnt/data10/t3fs14_cms/pool/meta] CleanerRun 3 ends on file 0x6 probe=false invokedFromDaemon=true finished=true fileDeleted=false nEntriesRead=26144 nINsObsolete=1208 nINsCleaned=7 nINsDead=7 nINsMigrated=0 nBINDeltasObsolete=555 nBINDeltasCleaned=0 nBINDeltasDead=0 nBINDeltasMigrated=0 nLNsObsolete=14013 nLNsCleaned=10359 nLNsDead=10215 nLNsMigrated=144 nLNsMarked=0 nLNQueueHits=2 nLNsLocked=0 logSummary=<CleanerLogSummary endFileNumAtLastAdjustment="0xe" initialAdjustments="3" recentLNSizesAndCounts="Cor:185141/6694-Est:2126328/10692 Cor:6987/503-Est:2916919/10376 "> inSummary=<INSummary totalINCount="1215" totalINSize="4270621" totalBINDeltaCount="555" totalBINDeltaSize="225008" obsoleteINCount="1208" obsoleteINSize="4247414" obsoleteBINDeltaCount="555" obsoleteBINDeltaSize="225008"/> estFileSummary=<summary totalCount="50507" totalSize="9999712" totalINCount="1215" totalINSize="4270621" totalLNCount="24372" totalLNSize="4724270" maxLNSize="0" obsoleteINCount="1212" obsoleteLNCount="14499" obsoleteLNSize="1807351" obsoleteLNSizeCounted="13996" getObsoleteSize="7213652" getObsoleteINSize="4260076" getObsoleteLNSize="1948755" getMaxObsoleteSize="7213652" getMaxObsoleteLNSize="1948755" getAvgObsoleteLNSizeNotCounted="281.12173"/> recalcFileSummary=<summary totalCount="50507" totalSize="9999712" totalINCount="1770" totalINSize="4495629" totalLNCount="24372" totalLNSize="4724270" maxLNSize="0" obsoleteINCount="1763" obsoleteLNCount="14013" obsoleteLNSize="1814338" obsoleteLNSizeCounted="14013" getObsoleteSize="7072000" getObsoleteINSize="4477849" getObsoleteLNSize="1814338" getMaxObsoleteSize="7072000" getMaxObsoleteLNSize="1814338" getAvgObsoleteLNSizeNotCounted="NaN"/> lnSizeCorrection=0.1390739 newLnSizeCorrection=0.111519866 estimatedUtilization=28 correctedUtilization=29 recalcUtilization=29 131108 17:05:47:854 INFO [/mnt/data10/t3fs14_cms/pool/meta] Chose lowest utilized file for cleaning. fileChosen: 0x9 totalUtilization: 49 bestFileUtilization: 30 lnSizeCorrectionFactor: 0.111519866 isProbe: false 131108 17:05:48:528 SEVERE [/mnt/data10/t3fs14_cms/pool/meta] %RED%com.sleepycat.je.EnvironmentFailureException: (JE 5.0.73) Environment must be closed, caused by: com.sleepycat.je.EnvironmentFailureException: Environment invalid because of previous exception: (JE 5.0.73) /mnt/data10/t3fs14_cms/pool/meta fetchTarget of 0xe/0x10f443 parent IN=80629 IN class=com.sleepycat.je.tree.BIN lastFullVersion=0xe/0x43b417 lastLoggedVersion=0xe/0x43b417 parent.getDirty()=true state=0 LOG_FILE_NOT_FOUND: Log file missing, log is likely invalid. Environment is invalid and must be closed.%ENDCOLOR% at com.sleepycat.je.EnvironmentFailureException.wrapSelf(EnvironmentFailureException.java:196) at com.sleepycat.je.dbi.EnvironmentImpl.checkIfInvalid(EnvironmentImpl.java:1512) at com.sleepycat.je.log.LogManager.serialLogInternal(LogManager.java:490) at com.sleepycat.je.log.SyncedLogManager.serialLog(SyncedLogManager.java:42) at com.sleepycat.je.log.LogManager.multiLog(LogManager.java:395) at com.sleepycat.je.log.LogManager.log(LogManager.java:335) at com.sleepycat.je.tree.LN.logInternal(LN.java:547) at com.sleepycat.je.tree.LN.log(LN.java:402) at com.sleepycat.je.cleaner.FileProcessor.processFoundLN(FileProcessor.java:1081) at com.sleepycat.je.cleaner.FileProcessor.processLN(FileProcessor.java:857) at com.sleepycat.je.cleaner.FileProcessor.processFile(FileProcessor.java:674) at com.sleepycat.je.cleaner.FileProcessor.doClean(FileProcessor.java:289) at com.sleepycat.je.cleaner.FileProcessor.onWakeup(FileProcessor.java:148) at com.sleepycat.je.utilint.DaemonThread.run(DaemonThread.java:163) at java.lang.Thread.run(Thread.java:722) Caused by: com.sleepycat.je.EnvironmentFailureException: Environment invalid because of previous exception: (JE 5.0.73) /mnt/data10/t3fs14_cms/pool/meta fetchTarget of 0xe/0x10f443 parent IN=80629 IN class=com.sleepycat.je.tree.BIN lastFullVersion=0xe/0x43b417 lastLoggedVersion=0xe/0x43b417 parent.getDirty()=true state=0 LOG_FILE_NOT_FOUND: Log file missing, log is likely invalid. Environment is invalid and must be closed. at com.sleepycat.je.tree.IN.fetchTarget(IN.java:1429) at com.sleepycat.je.tree.BIN.fetchTarget(BIN.java:1251) at com.sleepycat.je.dbi.CursorImpl.fetchCurrent(CursorImpl.java:2261) at com.sleepycat.je.dbi.CursorImpl.getCurrentAlreadyLatched(CursorImpl.java:1466) at com.sleepycat.je.dbi.CursorImpl.getNext(CursorImpl.java:1593) at com.sleepycat.je.Cursor.retrieveNextAllowPhantoms(Cursor.java:2924) at com.sleepycat.je.Cursor.retrieveNextNoDups(Cursor.java:2801) at com.sleepycat.je.Cursor.retrieveNext(Cursor.java:2775) at com.sleepycat.je.Cursor.getNextNoDup(Cursor.java:1244) at com.sleepycat.util.keyrange.RangeCursor.doGetNextNoDup(RangeCursor.java:920) at com.sleepycat.util.keyrange.RangeCursor.getNextNoDup(RangeCursor.java:475) at com.sleepycat.collections.DataCursor.getNextNoDup(DataCursor.java:456) at com.sleepycat.collections.BlockIterator.hasNext(BlockIterator.java:355) at org.dcache.pool.repository.meta.db.BerkeleyDBMetaDataRepository.list(BerkeleyDBMetaDataRepository.java:107) at org.dcache.pool.repository.ConsistentStore.list(ConsistentStore.java:111) at org.dcache.pool.repository.MetaDataCache.<init>(MetaDataCache.java:47) at org.dcache.pool.repository.v5.CacheRepositoryV5.init(CacheRepositoryV5.java:319) at org.dcache.pool.classic.PoolV4$1.run(PoolV4.java:437) Caused by: java.io.FileNotFoundException: /mnt/data10/t3fs14_cms/pool/meta/0000000e.jdb (Too many open files) at java.io.RandomAccessFile.open(Native Method) at java.io.RandomAccessFile.<init>(RandomAccessFile.java:233) at java.io.RandomAccessFile.<init>(RandomAccessFile.java:118) at com.sleepycat.je.log.FileManager$DefaultRandomAccessFile.<init>(FileManager.java:3209) at com.sleepycat.je.log.FileManager$6.createFile(FileManager.java:3237) at com.sleepycat.je.log.FileManager.openFileHandle(FileManager.java:1280) at com.sleepycat.je.log.FileManager.getFileHandle(FileManager.java:1189) at com.sleepycat.je.log.LogManager.getLogSource(LogManager.java:1141) at com.sleepycat.je.log.LogManager.getLogEntry(LogManager.java:845) at com.sleepycat.je.log.LogManager.getLogEntryAllowInvisibleAtRecovery(LogManager.java:809) at com.sleepycat.je.tree.IN.fetchTarget(IN.java:1412) ... 17 more 131108 17:05:48:529 INFO [/mnt/data10/t3fs14_cms/pool/meta] CleanerRun 1 ends on file 0x9 probe=false invokedFromDaemon=true finished=false fileDeleted=false nEntriesRead=0 nINsObsolete=198 nINsCleaned=1220 nINsDead=1220 nINsMigrated=0 nBINDeltasObsolete=0 nBINDeltasCleaned=0 nBINDeltasDead=0 nBINDeltasMigrated=0 nLNsObsolete=7129 nLNsCleaned=5729 nLNsDead=279 nLNsMigrated=5449 nLNsMarked=0 nLNQueueHits=519 nLNsLocked=0 logSummary=<CleanerLogSummary endFileNumAtLastAdjustment="0xe" initialAdjustments="3" recentLNSizesAndCounts="Cor:185141/6694-Est:2126328/10692 Cor:6987/503-Est:2916919/10376 "> inSummary=<INSummary totalINCount="1418" totalINSize="4688736" totalBINDeltaCount="0" totalBINDeltaSize="0" obsoleteINCount="198" obsoleteINSize="842086" obsoleteBINDeltaCount="0" obsoleteBINDeltaSize="0"/> estFileSummary=<summary totalCount="44984" totalSize="9998831" totalINCount="1539" totalINSize="5027283" totalLNCount="22212" totalLNSize="4291671" maxLNSize="0" obsoleteINCount="1592" obsoleteLNCount="12728" obsoleteLNSize="1285392" obsoleteLNSizeCounted="11154" getObsoleteSize="7420466" getObsoleteINSize="5027283" getObsoleteLNSize="1713306" getMaxObsoleteSize="7420466" getMaxObsoleteLNSize="1713306" getAvgObsoleteLNSizeNotCounted="271.86462"/> recalcFileSummary=<summary totalCount="26571" totalSize="7581969" totalINCount="1418" totalINSize="4688736" totalLNCount="12876" totalLNSize="2499948" maxLNSize="0" obsoleteINCount="198" obsoleteLNCount="7129" obsoleteLNSize="923168" obsoleteLNSizeCounted="7129" getObsoleteSize="1971156" getObsoleteINSize="654703" getObsoleteLNSize="923168" getMaxObsoleteSize="1971156" getMaxObsoleteLNSize="923168" getAvgObsoleteLNSizeNotCounted="NaN"/> lnSizeCorrection=0.111519866 newLnSizeCorrection=0.111519866 estimatedUtilization=26 correctedUtilization=30 recalcUtilization=74 ... </pre> dCache logs: <pre> 08 Nov 2013 17:05:45 (t3fs14_cms_5) [] Pool mode changed to disabled(fetch,store,stage,p2p-client,p2p-server): Initializing 08 Nov 2013 17:05:45 (t3fs14_cms_5) [] Reading inventory from [data=/mnt/data05/t3fs14_cms/pool/data;meta=/mnt/data05/t3fs14_cms/pool/meta] 08 Nov 2013 17:05:46 (t3fs14_cms_6) [] Pool mode changed to disabled(fetch,store,stage,p2p-client,p2p-server): Initializing 08 Nov 2013 17:05:46 (t3fs14_cms_6) [] Reading inventory from [data=/mnt/data07/t3fs14_cms/pool/data;meta=/mnt/data07/t3fs14_cms/pool/meta] 08 Nov 2013 17:05:46 (t3fs14_cms_7) [] Pool mode changed to disabled(fetch,store,stage,p2p-client,p2p-server): Initializing 08 Nov 2013 17:05:47 (t3fs14_cms_7) [] Reading inventory from [data=/mnt/data08/t3fs14_cms/pool/data;meta=/mnt/data08/t3fs14_cms/pool/meta] 08 Nov 2013 17:05:47 (t3fs14_cms_8) [] Pool mode changed to disabled(fetch,store,stage,p2p-client,p2p-server): Initializing 08 Nov 2013 17:05:47 (t3fs14_cms_8) [] Reading inventory from [data=/mnt/data09/t3fs14_cms/pool/data;meta=/mnt/data09/t3fs14_cms/pool/meta] 08 Nov 2013 17:05:48 (t3fs14_cms_9) [] Pool mode changed to disabled(fetch,store,stage,p2p-client,p2p-server): Initializing 08 Nov 2013 17:05:48 (t3fs14_cms_9) [] Reading inventory from [data=/mnt/data10/t3fs14_cms/pool/data;meta=/mnt/data10/t3fs14_cms/pool/meta] 08 Nov 2013 17:05:48 (t3fs14_cms_9) [] Pool restart required due to Berkeley DB failure: (JE 5.0.73) Environment must be closed, caused by: com.sleepycat.je.EnvironmentFailureException: Environment invalid because of previous exception: (JE 5.0.73) /mnt/data10/t3fs14_cms/pool/meta fetchTarget of 0xe/0x10f443 parent IN=80629 IN class=com.sleepycat.je.tree.BIN lastFullVersion=0xe/0x43b417 lastLoggedVersion=0xe/0x43b417 parent.getDirty()=true state=0 LOG_FILE_NOT_FOUND: Log file missing, log is likely invalid. Environment is invalid and must be closed. Nov 08, 2013 5:05:48 PM com.sleepycat.je.utilint.DaemonThread run SEVERE: <DaemonThread name="Cleaner-1"/> caught exception, com.sleepycat.je.EnvironmentFailureException: (JE 5.0.73) Environment must be closed, caused by: com.sleepycat.je.EnvironmentFailureException: Environment invalid because of previous exception: (JE 5.0.73) /mnt/data10/t3fs14_cms/pool/meta fetchTarget of 0xe/0x10f443 parent IN=80629 IN class=com.sleepycat.je.tree.BIN lastFullVersion=0xe/0x43b417 lastLoggedVersion=0xe/0x43b417 parent.getDirty()=true state=0 LOG_FILE_NOT_FOUND: Log file missing, log is likely invalid. Environment is invalid and must be closed. Exiting com.sleepycat.je.EnvironmentFailureException: (JE 5.0.73) Environment must be closed, caused by: com.sleepycat.je.EnvironmentFailureException: Environment invalid because of previous exception: (JE 5.0.73) /mnt/data10/t3fs14_cms/pool/meta fetchTarget of 0xe/0x10f443 parent IN=80629 IN class=com.sleepycat.je.tree.BIN lastFullVersion=0xe/0x43b417 lastLoggedVersion=0xe/0x43b417 parent.getDirty()=true state=0 LOG_FILE_NOT_FOUND: Log file missing, log is likely invalid. Environment is invalid and must be closed. at com.sleepycat.je.EnvironmentFailureException.wrapSelf(EnvironmentFailureException.java:196) at com.sleepycat.je.dbi.EnvironmentImpl.checkIfInvalid(EnvironmentImpl.java:1512) at com.sleepycat.je.log.LogManager.serialLogInternal(LogManager.java:490) at com.sleepycat.je.log.SyncedLogManager.serialLog(SyncedLogManager.java:42) at com.sleepycat.je.log.LogManager.multiLog(LogManager.java:395) at com.sleepycat.je.log.LogManager.log(LogManager.java:335) at com.sleepycat.je.tree.LN.logInternal(LN.java:547) at com.sleepycat.je.tree.LN.log(LN.java:402) at com.sleepycat.je.cleaner.FileProcessor.processFoundLN(FileProcessor.java:1081) at com.sleepycat.je.cleaner.FileProcessor.processLN(FileProcessor.java:857) at com.sleepycat.je.cleaner.FileProcessor.processFile(FileProcessor.java:674) at com.sleepycat.je.cleaner.FileProcessor.doClean(FileProcessor.java:289) at com.sleepycat.je.cleaner.FileProcessor.onWakeup(FileProcessor.java:148) at com.sleepycat.je.utilint.DaemonThread.run(DaemonThread.java:163) at java.lang.Thread.run(Thread.java:722) Caused by: com.sleepycat.je.EnvironmentFailureException: Environment invalid because of previous exception: (JE 5.0.73) /mnt/data10/t3fs14_cms/pool/meta fetchTarget of 0xe/0x10f443 parent IN=80629 IN class=com.sleepycat.je.tree.BIN lastFullVersion=0xe/0x43b417 lastLoggedVersion=0xe/0x43b417 parent.getDirty()=true state=0 LOG_FILE_NOT_FOUND: Log file missing, log is likely invalid. Environment is invalid and must be closed. at com.sleepycat.je.tree.IN.fetchTarget(IN.java:1429) at com.sleepycat.je.tree.BIN.fetchTarget(BIN.java:1251) at com.sleepycat.je.dbi.CursorImpl.fetchCurrent(CursorImpl.java:2261) at com.sleepycat.je.dbi.CursorImpl.getCurrentAlreadyLatched(CursorImpl.java:1466) at com.sleepycat.je.dbi.CursorImpl.getNext(CursorImpl.java:1593) at com.sleepycat.je.Cursor.retrieveNextAllowPhantoms(Cursor.java:2924) at com.sleepycat.je.Cursor.retrieveNextNoDups(Cursor.java:2801) at com.sleepycat.je.Cursor.retrieveNext(Cursor.java:2775) at com.sleepycat.je.Cursor.getNextNoDup(Cursor.java:1244) at com.sleepycat.util.keyrange.RangeCursor.doGetNextNoDup(RangeCursor.java:920) at com.sleepycat.util.keyrange.RangeCursor.getNextNoDup(RangeCursor.java:475) at com.sleepycat.collections.DataCursor.getNextNoDup(DataCursor.java:456) at com.sleepycat.collections.BlockIterator.hasNext(BlockIterator.java:355) at org.dcache.pool.repository.meta.db.BerkeleyDBMetaDataRepository.list(BerkeleyDBMetaDataRepository.java:107) at org.dcache.pool.repository.ConsistentStore.list(ConsistentStore.java:111) at org.dcache.pool.repository.MetaDataCache.<init>(MetaDataCache.java:47) at org.dcache.pool.repository.v5.CacheRepositoryV5.init(CacheRepositoryV5.java:319) at org.dcache.pool.classic.PoolV4$1.run(PoolV4.java:437) Caused by: java.io.FileNotFoundException: /mnt/data10/t3fs14_cms/pool/meta/0000000e.jdb (Too many open files) at java.io.RandomAccessFile.open(Native Method) at java.io.RandomAccessFile.<init>(RandomAccessFile.java:233) at java.io.RandomAccessFile.<init>(RandomAccessFile.java:118) at com.sleepycat.je.log.FileManager$DefaultRandomAccessFile.<init>(FileManager.java:3209) at com.sleepycat.je.log.FileManager$6.createFile(FileManager.java:3237) at com.sleepycat.je.log.FileManager.openFileHandle(FileManager.java:1280) at com.sleepycat.je.log.FileManager.getFileHandle(FileManager.java:1189) at com.sleepycat.je.log.LogManager.getLogSource(LogManager.java:1141) at com.sleepycat.je.log.LogManager.getLogEntry(LogManager.java:845) at com.sleepycat.je.log.LogManager.getLogEntryAllowInvisibleAtRecovery(LogManager.java:809) at com.sleepycat.je.tree.IN.fetchTarget(IN.java:1412) ... 17 more (666) file:/usr/share/dcache/services/pool.batch: line 95: (3) java.lang.RuntimeException: dmg.util.CommandThrowableException: (3) Failed to create bean 'pool' : Too many open files from ac_create_$_2_3 08 Nov 2013 17:05:48 (t3fs14_cms_9) [] Repository reported a problem. Please report this to support@dcache.org. com.sleepycat.je.EnvironmentFailureException: (JE 5.0.73) Environment must be closed, caused by: com.sleepycat.je.EnvironmentFailureException: Environment invalid because of previous exception: (JE 5.0.73) /mnt/data10/t3fs14_cms/pool/meta fetchTarget of 0xe/0x10f443 parent IN=80629 IN class=com.sleepycat.je.tree.BIN lastFullVersion=0xe/0x43b417 lastLoggedVersion=0xe/0x43b417 parent.getDirty()=true state=0 LOG_FILE_NOT_FOUND: Log file missing, log is likely invalid. Environment is invalid and must be closed. at com.sleepycat.je.EnvironmentFailureException.wrapSelf(EnvironmentFailureException.java:196) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.je.dbi.EnvironmentImpl.checkIfInvalid(EnvironmentImpl.java:1512) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.je.dbi.CursorImpl.checkEnv(CursorImpl.java:2838) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.je.Cursor.checkEnv(Cursor.java:4048) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.je.Cursor.close(Cursor.java:515) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.collections.CurrentTransaction.closeCursor(CurrentTransaction.java:484) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.collections.DataCursor.close(DataCursor.java:215) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.collections.StoredContainer.closeCursor(StoredContainer.java:422) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.collections.BlockIterator.hasNext(BlockIterator.java:376) ~[je-5.0.73.jar:5.0.73] at org.dcache.pool.repository.meta.db.BerkeleyDBMetaDataRepository.list(BerkeleyDBMetaDataRepository.java:107) ~[dcache-core-2.6.12.jar:2.6.12] at org.dcache.pool.repository.ConsistentStore.list(ConsistentStore.java:111) ~[dcache-core-2.6.12.jar:2.6.12] at org.dcache.pool.repository.MetaDataCache.<init>(MetaDataCache.java:47) ~[dcache-core-2.6.12.jar:2.6.12] at org.dcache.pool.repository.v5.CacheRepositoryV5.init(CacheRepositoryV5.java:319) ~[dcache-core-2.6.12.jar:2.6.12] at org.dcache.pool.classic.PoolV4$1.run(PoolV4.java:437) ~[dcache-core-2.6.12.jar:2.6.12] Caused by: com.sleepycat.je.EnvironmentFailureException: Environment invalid because of previous exception: (JE 5.0.73) /mnt/data10/t3fs14_cms/pool/meta fetchTarget of 0xe/0x10f443 parent IN=80629 IN class=com.sleepycat.je.tree.BIN lastFullVersion=0xe/0x43b417 lastLoggedVersion=0xe/0x43b417 parent.getDirty()=true state=0 LOG_FILE_NOT_FOUND: Log file missing, log is likely invalid. Environment is invalid and must be closed. at com.sleepycat.je.tree.IN.fetchTarget(IN.java:1429) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.je.tree.BIN.fetchTarget(BIN.java:1251) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.je.dbi.CursorImpl.fetchCurrent(CursorImpl.java:2261) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.je.dbi.CursorImpl.getCurrentAlreadyLatched(CursorImpl.java:1466) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.je.dbi.CursorImpl.getNext(CursorImpl.java:1593) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.je.Cursor.retrieveNextAllowPhantoms(Cursor.java:2924) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.je.Cursor.retrieveNextNoDups(Cursor.java:2801) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.je.Cursor.retrieveNext(Cursor.java:2775) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.je.Cursor.getNextNoDup(Cursor.java:1244) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.util.keyrange.RangeCursor.doGetNextNoDup(RangeCursor.java:920) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.util.keyrange.RangeCursor.getNextNoDup(RangeCursor.java:475) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.collections.DataCursor.getNextNoDup(DataCursor.java:456) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.collections.BlockIterator.hasNext(BlockIterator.java:355) ~[je-5.0.73.jar:5.0.73] ... 5 common frames omitted Caused by: java.io.FileNotFoundException: /mnt/data10/t3fs14_cms/pool/meta/0000000e.jdb (Too many open files) at java.io.RandomAccessFile.open(Native Method) ~[na:1.7.0_21] at java.io.RandomAccessFile.<init>(RandomAccessFile.java:233) ~[na:1.7.0_21] at java.io.RandomAccessFile.<init>(RandomAccessFile.java:118) ~[na:1.7.0_21] at com.sleepycat.je.log.FileManager$DefaultRandomAccessFile.<init>(FileManager.java:3209) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.je.log.FileManager$6.createFile(FileManager.java:3237) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.je.log.FileManager.openFileHandle(FileManager.java:1280) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.je.log.FileManager.getFileHandle(FileManager.java:1189) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.je.log.LogManager.getLogSource(LogManager.java:1141) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.je.log.LogManager.getLogEntry(LogManager.java:845) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.je.log.LogManager.getLogEntryAllowInvisibleAtRecovery(LogManager.java:809) ~[je-5.0.73.jar:5.0.73] at com.sleepycat.je.tree.IN.fetchTarget(IN.java:1412) ~[je-5.0.73.jar:5.0.73] ... 17 common frames omitted 08 Nov 2013 17:05:48 (System) [] Failure at startup: (666) file:/usr/share/dcache/services/pool.batch: line 95: (3) java.lang.RuntimeException: dmg.util.CommandThrowableException: (3) Failed to create bean 'pool' : Too many open files from ac_create_$_2_3 08 Nov 2013 17:05:48 (t3fs14_cms_9) [] Pool not enabled t3fs14_cms_9 08 Nov 2013 17:05:48 (t3fs14_cms_9) [] Pool mode changed to disabled(fetch,store,stage,p2p-client,p2p-server,dead): Init failed: (JE 5.0.73) Envir </pre>
UpgradePlanningForm
Title
dCache 2.6 upgrade
Summary
2.2 to 2.6 upgrade
Target Date
08. 11. 2013
This topic: CmsTier3
>
WebHome
>
AdminArea
>
UpgradePlanning201311081851
Topic revision: r3 - 2013-11-11 - FabioMartinelli
Copyright © 2008-2024 by the contributing authors. All material on this collaboration platform is the property of the contributing authors.
Ideas, requests, problems regarding TWiki?
Send feedback