java - Quartz Scheduler worker threads not running jobs even with huge number of jobs in queue -

we have 30 worker threads in quartz scheduler, , there 104k jobs in queue (they piled since quartz not processing fast enough , lots of bulk jobs submitted). can see in thread dump, of these threads waiting without working. why these worker threads not running jobs in queue missing fire time?

could there bottlenecks before jobs distributed workers? see few jobs running @ time instead of upto 30 jobs @ once.

i should note happens irrespective of 1 scheduler process or many.

any pointers find why many (at 30) jobs not getting picked , executed @ once appreciated.

quartz version: 2.1.3 (with oracle jdbc jobstore)

config file

#============================================================================ # configure main scheduler properties #============================================================================  org.quartz.scheduler.instancename = ****_scheduler org.quartz.scheduler.instanceid = auto  #============================================================================ # configure threadpool #============================================================================  org.quartz.threadpool.class = org.quartz.simpl.simplethreadpool org.quartz.threadpool.threadcount = 30 org.quartz.threadpool.threadpriority = 5   #============================================================================ # configure jobstore #============================================================================  org.quartz.jobstore.class = org.quartz.impl.jdbcjobstore.jobstoretx org.quartz.jobstore.datasource = myds org.quartz.jobstore.driverdelegateclass = org.quartz.impl.jdbcjobstore.oracle.oracledelegate org.quartz.jobstore.isclustered=true org.quartz.jobstore.clustercheckininterval = 20000 org.quartz.jobstore.tableprefix = qrtz_   #============================================================================ # configure datasources #============================================================================  org.quartz.datasource.myds.driver = oracle.jdbc.driver.oracledriver org.quartz.datasource.myds.url = jdbc:oracle:thin:@***** org.quartz.datasource.myds.user = ***** #org.quartz.datasource.myds.password = ***** org.quartz.datasource.myds.maxconnections = 10 org.quartz.datasource.myds.validationquery=select 0 dual  #============================================================================ # configured batch trigger acquisition #============================================================================ org.quartz.scheduler.skipupdatecheck = false org.quartz.scheduler.batchtriggeracquisitionmaxcount = 30 org.quartz.jobstore.acquiretriggerswithinlock = true ~ 

thread dump

i see quartzschedulerthread (i assume manager thread?) , misfirehandler threads running same stacktrace (except locked monitor id).

stacktrace quartzschedulerthread

at java.net.socketinputstream.socketread0(native method)  @ java.net.socketinputstream.read(socketinputstream.java:129)  @ oracle.net.ns.packet.receive(packet.java:300)  @ oracle.net.ns.datapacket.receive(datapacket.java:106)  @ oracle.net.ns.netinputstream.getnextpacket(netinputstream.java:315)  @ oracle.net.ns.netinputstream.read(netinputstream.java:260)  @ oracle.net.ns.netinputstream.read(netinputstream.java:185)  @ oracle.net.ns.netinputstream.read(netinputstream.java:102)  @ oracle.jdbc.driver.t4csocketinputstreamwrapper.readnextpacket(t4csocketinputstreamwrapper.java:124)  @ oracle.jdbc.driver.t4csocketinputstreamwrapper.read(t4csocketinputstreamwrapper.java:80)  @ oracle.jdbc.driver.t4cmarengine.unmarshalub1(t4cmarengine.java:1137)  @ oracle.jdbc.driver.t4cttifun.receive(t4cttifun.java:290)  @ oracle.jdbc.driver.t4cttifun.dorpc(t4cttifun.java:192)  @ oracle.jdbc.driver.t4c8oall.dooall(t4c8oall.java:531)  @ oracle.jdbc.driver.t4cpreparedstatement.dooall8(t4cpreparedstatement.java:207)  @ oracle.jdbc.driver.t4cpreparedstatement.executeforrows(t4cpreparedstatement.java:1044)  @ oracle.jdbc.driver.oraclestatement.executemaybedescribe(oraclestatement.java:1199)  @ oracle.jdbc.driver.oraclestatement.doexecutewithtimeout(oraclestatement.java:1289)  @ oracle.jdbc.driver.oraclepreparedstatement.executeinternal(oraclepreparedstatement.java:3584)  @ oracle.jdbc.driver.oraclepreparedstatement.executequery(oraclepreparedstatement.java:3628)  - locked [0x00000007f5d0fd30] (a oracle.jdbc.driver.t4cconnection)  @ oracle.jdbc.driver.oraclepreparedstatementwrapper.executequery(oraclepreparedstatementwrapper.java:1493)  @ com.mchange.v2.c3p0.impl.newproxypreparedstatement.executequery(newproxypreparedstatement.java:76)  @ org.quartz.impl.jdbcjobstore.stdrowlocksemaphore.executesql(stdrowlocksemaphore.java:91)  @ org.quartz.impl.jdbcjobstore.dbsemaphore.obtainlock(dbsemaphore.java:115)  @ org.quartz.impl.jdbcjobstore.jobstoresupport.executeinnonmanagedtxlock(jobstoresupport.java:3803)  @ org.quartz.impl.jdbcjobstore.jobstoresupport.triggersfired(jobstoresupport.java:2893)  @ org.quartz.core.quartzschedulerthread.run(quartzschedulerthread.java:326) 

stacktrace misfirehandler thread:

at java.net.socketinputstream.socketread0(native method)  @ java.net.socketinputstream.read(socketinputstream.java:129)  @ oracle.net.ns.packet.receive(packet.java:300)  @ oracle.net.ns.datapacket.receive(datapacket.java:106)  @ oracle.net.ns.netinputstream.getnextpacket(netinputstream.java:315)  @ oracle.net.ns.netinputstream.read(netinputstream.java:260)  @ oracle.net.ns.netinputstream.read(netinputstream.java:185)  @ oracle.net.ns.netinputstream.read(netinputstream.java:102)  @ oracle.jdbc.driver.t4csocketinputstreamwrapper.readnextpacket(t4csocketinputstreamwrapper.java:124)  @ oracle.jdbc.driver.t4csocketinputstreamwrapper.read(t4csocketinputstreamwrapper.java:80)  @ oracle.jdbc.driver.t4cmarengine.unmarshalub1(t4cmarengine.java:1137)  @ oracle.jdbc.driver.t4cttifun.receive(t4cttifun.java:290)  @ oracle.jdbc.driver.t4cttifun.dorpc(t4cttifun.java:192)  @ oracle.jdbc.driver.t4c8oall.dooall(t4c8oall.java:531)  @ oracle.jdbc.driver.t4cpreparedstatement.dooall8(t4cpreparedstatement.java:207)  @ oracle.jdbc.driver.t4cpreparedstatement.executeforrows(t4cpreparedstatement.java:1044)  @ oracle.jdbc.driver.oraclestatement.executemaybedescribe(oraclestatement.java:1199)  @ oracle.jdbc.driver.oraclestatement.doexecutewithtimeout(oraclestatement.java:1289)  @ oracle.jdbc.driver.oraclepreparedstatement.executeinternal(oraclepreparedstatement.java:3584)  @ oracle.jdbc.driver.oraclepreparedstatement.executequery(oraclepreparedstatement.java:3628)  - locked [0x00000007f58c8010] (a oracle.jdbc.driver.t4cconnection)  @ oracle.jdbc.driver.oraclepreparedstatementwrapper.executequery(oraclepreparedstatementwrapper.java:1493)  @ com.mchange.v2.c3p0.impl.newproxypreparedstatement.executequery(newproxypreparedstatement.java:76)  @ org.quartz.impl.jdbcjobstore.stdjdbcdelegate.hasmisfiredtriggersinstate(stdjdbcdelegate.java:379)  @ org.quartz.impl.jdbcjobstore.jobstoresupport.recovermisfiredjobs(jobstoresupport.java:946)  @ org.quartz.impl.jdbcjobstore.jobstoresupport.dorecovermisfires(jobstoresupport.java:3187)  @ org.quartz.impl.jdbcjobstore.jobstoresupport$misfirehandler.manage(jobstoresupport.java:3946)  @ org.quartz.impl.jdbcjobstore.jobstoresupport$misfirehandler.run(jobstoresupport.java:3967)