High AJP and heap usage after adding 3rd node

[btlfed7]

Any hints or places to look why adding a 3rd node would cause my application to crash?

I was kinda thrown into a broken jboss application because my background in weblogic, so please go easy on me.

Let me know what additional info you need, but here is what I have.

Server: name=localhost port=80
Time: datetime=20100202133948 tz=Central Standard Time unix=1265139588
Software: web_server="Apache/2.0.63 (Win32) mod_jk/1.2.28" jk_version=mod_jk/1.2.28
Balancer Workers: count=1
Balancer Worker: name=loadbalancer type=lb sticky_session=True sticky_session_force=False retries=2 recover_time=60 error_escalation_time=30 max_reply_timeouts=0 method=Sessions lock=Optimistic member_count=3 good=3 degraded=0 bad=0 busy=132 max_busy=169 map_count=12 time_to_maintenance_min=44 time_to_maintenance_max=106 last_reset_at=1265113731 last_reset_ago=25857
Member: name=node1 type=ajp13 host=10.16.72.67 port=8109 address=10.16.72.67:8109 connection_pool_timeout=0 ping_timeout=10000 connect_timeout=0 prepost_timeout=0 reply_timeout=0 retries=2 connection_ping_interval=0 recovery_options=0 max_packet_size=8192 activation=ACT lbfactor=1 route="node1" redirect="" domain="" distance=0 state=OK lbmult=1 lbvalue=0 elected=662 errors=0 client_errors=6 reply_timeouts=0 transferred=384578 read=4878199 busy=3 max_busy=18 time_to_recover_min=0 time_to_recover_max=0 last_reset_at=1265137797 last_reset_ago=1791
Member: name=node2 type=ajp13 host=10.16.72.67 port=8209 address=10.16.72.67:8209 connection_pool_timeout=0 ping_timeout=10000 connect_timeout=0 prepost_timeout=0 reply_timeout=0 retries=2 connection_ping_interval=0 recovery_options=0 max_packet_size=8192 activation=ACT lbfactor=1 route="node2" redirect="" domain="" distance=0 state=OK lbmult=1 lbvalue=0 elected=6552 errors=0 client_errors=18 reply_timeouts=0 transferred=4102522 read=53048967 busy=60 max_busy=103 time_to_recover_min=0 time_to_recover_max=0 last_reset_at=1265137455 last_reset_ago=2133
Member: name=node3 type=ajp13 host=10.16.72.67 port=8309 address=10.16.72.67:8309 connection_pool_timeout=0 ping_timeout=10000 connect_timeout=0 prepost_timeout=0 reply_timeout=0 retries=2 connection_ping_interval=0 recovery_options=0 max_packet_size=8192 activation=ACT lbfactor=1 route="node3" redirect="" domain="" distance=0 state=OK/IDLE lbmult=1 lbvalue=0 elected=18 errors=0 client_errors=0 reply_timeouts=0 transferred=5519 read=5040 busy=51 max_busy=52 time_to_recover_min=0 time_to_recover_max=0 last_reset_at=1265137461 last_reset_ago=2127
Map: id=1 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid/zeroclientAdmin/*" type="Wildchar" source="uriworkermap" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0"
Map: id=2 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid/zeroclientAdmin/*" type="Wildchar" source="JkMount" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0"
Map: id=3 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid/zeroclient/*" type="Wildchar" source="uriworkermap" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0"
Map: id=4 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid/zeroclient/*" type="Wildchar" source="JkMount" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0"
Map: id=5 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid/zeroclientAdmin*" type="Wildchar" source="uriworkermap" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0"
Map: id=6 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid/zeroclientAdmin*" type="Wildchar" source="JkMount" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0"
Map: id=7 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid/zeroclient*" type="Wildchar" source="uriworkermap" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0"
Map: id=8 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid/zeroclient*" type="Wildchar" source="JkMount" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0"
Map: id=9 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid/*" type="Wildchar" source="uriworkermap" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0"
Map: id=10 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid/*" type="Wildchar" source="JkMount" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0"
Map: id=11 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid*" type="Wildchar" source="uriworkermap" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0"
Map: id=12 server="USSTLZ-PFINWI05.emrsn.org:80" uri="/getpaid*" type="Wildchar" source="JkMount" reply_timeout="-1" fail_on_status="" active="" disabled="" stopped="" use_server_errors="0"
AJP Workers: count=3
AJP Worker: name=node1 type=ajp13 host=10.16.72.67 port=8109 address=10.16.72.67:8109 connection_pool_timeout=0 ping_timeout=10000 connect_timeout=0 prepost_timeout=0 reply_timeout=0 retries=2 connection_ping_interval=0 recovery_options=0 max_packet_size=8192 used=0 errors=0 client_errors=0 reply_timeouts=0 transferred=0 read=0 busy=0 max_busy=0 map_count=0 last_reset_at=1265113731 last_reset_ago=25857
AJP Worker: name=node2 type=ajp13 host=10.16.72.67 port=8209 address=10.16.72.67:8209 connection_pool_timeout=0 ping_timeout=10000 connect_timeout=0 prepost_timeout=0 reply_timeout=0 retries=2 connection_ping_interval=0 recovery_options=0 max_packet_size=8192 used=0 errors=0 client_errors=0 reply_timeouts=0 transferred=0 read=0 busy=0 max_busy=0 map_count=0 last_reset_at=1265113731 last_reset_ago=25857
AJP Worker: name=node3 type=ajp13 host=10.16.72.67 port=8309 address=10.16.72.67:8309 connection_pool_timeout=0 ping_timeout=10000 connect_timeout=0 prepost_timeout=0 reply_timeout=0 retries=2 connection_ping_interval=0 recovery_options=0 max_packet_size=8192 used=0 errors=0 client_errors=0 reply_timeouts=0 transferred=0 read=0 busy=0 max_busy=0 map_count=0 last_reset_at=1265113731 last_reset_ago=25857
Result: type=OK message="Action finished"



Here are my configs as they stand now:



workers.properties:



# Define list of workers that will be used
# for mapping requests
worker.list=node1,node2,node3,loadbalancer,status
# Define Node1
# modify the host as your host IP or DNS name.
worker.node1.port=8109
worker.node1.host=10.16.72.67
worker.node1.type=ajp13
worker.node1.lbfactor=1
worker.node1.cachesize=100
# Define Node2
# modify the host as your host IP or DNS name.
worker.node2.port=8209
worker.node2.host=10.16.72.67
worker.node2.type=ajp13
worker.node2.lbfactor=1
worker.node2.cachesize=100
# Define Node3
# modify the host as your host IP or DNS name.
worker.node3.port=8309
worker.node3.host=10.16.72.67
worker.node3.type=ajp13
worker.node3.lbfactor=1
worker.node3.cachesize=100
# Load-balancing behaviour
worker.loadbalancer.type=lb
worker.loadbalancer.balanced_workers=node1,node2,n ode3
worker.loadbalancer.sticky_session=1
#worker.list=loadbalancer
# Status worker for managing load balancer
worker.status.type=status



uriworkermap.properties:



# Simple worker configuration file
# Mount the Servlet context to the ajp13 worker
/getpaid/zeroclient*=loadbalancer
/getpaid/zeroclient/*=loadbalancer
/getpaid/zeroclientAdmin*=loadbalancer
/getpaid/zeroclientAdmin/*=loadbalancer
/getpaid/*=loadbalancer
/getpaid*=loadbalancer



mod_jk.conf



# Load mod_jk module
# Specify the filename of the mod_jk lib
LoadModule jk_module modules/mod_jk.so

JkWorkersFile conf/workers.properties

JkLogFile logs/mod_jk.log

JkLogLevel info

JkLogStampFormat "[%a %b %d %H:%M:%S %Y]"

JkOptions +ForwardKeySize +ForwardURICompat -ForwardDirectories

JkRequestLogFormat "%w %V %T"

JkMount /getpaid/zeroclient* loadbalancer
JkMount /getpaid/zeroclient/* loadbalancer
JkMount /getpaid/zeroclientAdmin* loadbalancer
JkMount /getpaid/zeroclientAdmin/* loadbalancer
JkMount /getpaid/* loadbalancer
JkMount /getpaid* loadbalancer


JkMountFile conf/uriworkermap.properties
JkShmFile logs/jk.shm



JkMount status
Order deny,allow
Deny from all
Allow from 127.0.0.1


changes only from the httpd.conf:



UseCanonicalName Off

ServerSignature On
Win32DisableAcceptEx



Include conf/mod-jk.conf