Topics covered in this article are:
Problem Statement
Alerts are not generated on a Director OnPrem setup.
Symptoms
No alerts will be generated. This occurs when Cassandra is deployed on the cluster.
Troubleshooting
Execute the following command:
kubectl get cm -n <director_namespace>
Sample Output:
NAME DATA AGE
cassandra-config 1 13m
directoronprem-nginx-ingress-controller 7 13m
elastalert-config 1 13m
elastalert-rules 2 13m
elasticsearch-curator-action 1 8m14s
elasticsearch-curator-config 1 8m13s
elasticsearch-internalusers 1 13m
ingress-controller-leader-nginx 0 12m
maya-config 1 13m
maya-grafana-cfgmap 1 13m
mysql 2 13m
Next, you need to edit the cassandra config map
kubectl get cm -oyaml -n director cassandra-config
Sample output:
apiVersion: v1
data:
run.sh: "#!/bin/bash\n\n# Copyright 2016 The Kubernetes Authors.\n#\n# Licensed
under the Apache License, Version 2.0 (the \"License\");\n# you may not use this
file except in compliance with the License.\n# You may obtain a copy of the License
at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required
by applicable law or agreed to in writing, software\n# distributed under the License
is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.\n# See the License for the specific language
governing permissions and\n# limitations under the License.\n\nset -e\nCASSANDRA_CONF_DIR=/etc/cassandra\nCASSANDRA_CFG=$CASSANDRA_CONF_DIR/cassandra.yaml\n\n#
we are doing StatefulSet or just setting our seeds\nif [ -z \"$CASSANDRA_SEEDS\"
]; then\n HOSTNAME=$(hostname -f)\n CASSANDRA_SEEDS=$(hostname -f)\nfi\n\n#
The following vars relate to there counter parts in $CASSANDRA_CFG\n# for instance
rpc_address\nCASSANDRA_RPC_ADDRESS=\"${CASSANDRA_RPC_ADDRESS:-0.0.0.0}\"\nCASSANDRA_NUM_TOKENS=\"${CASSANDRA_NUM_TOKENS:-32}\"\nCASSANDRA_CLUSTER_NAME=\"${CASSANDRA_CLUSTER_NAME:='Test
Cluster'}\"\nCASSANDRA_LISTEN_ADDRESS=${POD_IP:-$HOSTNAME}\nCASSANDRA_BROADCAST_ADDRESS=${POD_IP:-$HOSTNAME}\nCASSANDRA_BROADCAST_RPC_ADDRESS=${POD_IP:-$HOSTNAME}\nCASSANDRA_DISK_OPTIMIZATION_STRATEGY=\"${CASSANDRA_DISK_OPTIMIZATION_STRATEGY:-ssd}\
"\nCASSANDRA_MIGRATION_WAIT=\"${CASSANDRA_MIGRATION_WAIT:-1}\"\nCASSANDRA_ENDPOINT_SNITCH=\"${CASSANDRA_ENDPOINT_SNITCH:-SimpleSnitch}\"\nCASSANDRA_DC=\"${CASSANDRA_DC}\"\nCASSANDRA_RACK=\"${CASSANDRA_RACK}\
"\nCASSANDRA_RING_DELAY=\"${CASSANDRA_RING_DELAY:-30000}\"\nCASSANDRA_AUTO_BOOTSTRAP=\"${CASSANDRA_AUTO_BOOTSTRAP:-true}\"\nCASSANDRA_SEEDS=\"${CASSANDRA_SEEDS:false}\"\nCASSANDRA_SEED_PROVIDER=\"${CASSANDRA_SEED_PROVIDER:-org.apache.cassandra.locator.SimpleSeedProvider}\"\nCASSANDRA_AUTO_BOOTSTRAP=\"${CASSANDRA_AUTO_BOOTSTRAP:false}\
"\nwrite_request_timeout_in_ms=\"${write_request_timeout_in_ms:-3000}\
"\nread_request_timeout_in_ms=\"${read_request_timeout_in_ms:-5000}\"\n\n#
Turn off JMX auth\nCASSANDRA_OPEN_JMX=\"${CASSANDRA_OPEN_JMX:-false}\"\n# send
GC to STDOUT\nCASSANDRA_GC_STDOUT=\"${CASSANDRA_GC_STDOUT:-false}\"\n\necho Starting
Cassandra on ${CASSANDRA_LISTEN_ADDRESS}\necho CASSANDRA_CONF_DIR ${CASSANDRA_CONF_DIR}\necho
CASSANDRA_CFG ${CASSANDRA_CFG}\necho write_request_timeout_in_ms ${write_request_timeout_in_ms}\necho
read_request_timeout_in_ms ${read_request_timeout_in_ms}\necho CASSANDRA_AUTO_BOOTSTRAP
${CASSANDRA_AUTO_BOOTSTRAP}\necho CASSANDRA_BROADCAST_ADDRESS ${CASSANDRA_BROADCAST_ADDRESS}\necho
CASSANDRA_BROADCAST_RPC_ADDRESS ${CASSANDRA_BROADCAST_RPC_ADDRESS}\necho CASSANDRA_CLUSTER_NAME
${CASSANDRA_CLUSTER_NAME}\necho CASSANDRA_COMPACTION_THROUGHPUT_MB_PER_SEC ${CASSANDRA_COMPACTION_THROUGHPUT_MB_PER_SEC}\necho
CASSANDRA_CONCURRENT_COMPACTORS ${CASSANDRA_CONCURRENT_COMPACTORS}\necho CASSANDRA_CONCURRENT_READS
${CASSANDRA_CONCURRENT_READS}\necho CASSANDRA_CONCURRENT_WRITES ${CASSANDRA_CONCURRENT_WRITES}\necho
CASSANDRA_COUNTER_CACHE_SIZE_IN_MB ${CASSANDRA_COUNTER_CACHE_SIZE_IN_MB}\necho
CASSANDRA_DC ${CASSANDRA_DC}\necho CASSANDRA_DISK_OPTIMIZATION_STRATEGY ${CASSANDRA_DISK_OPTIMIZATION_STRATEGY}\necho
CASSANDRA_ENDPOINT_SNITCH ${CASSANDRA_ENDPOINT_SNITCH}\necho CASSANDRA_GC_WARN_THRESHOLD_IN_MS
${CASSANDRA_GC_WARN_THRESHOLD_IN_MS}\necho CASSANDRA_INTERNODE_COMPRESSION ${CASSANDRA_INTERNODE_COMPRESSION}\necho
CASSANDRA_KEY_CACHE_SIZE_IN_MB ${CASSANDRA_KEY_CACHE_SIZE_IN_MB}\necho CASSANDRA_LISTEN_ADDRESS
${CASSANDRA_LISTEN_ADDRESS}\necho CASSANDRA_LISTEN_INTERFACE ${CASSANDRA_LISTEN_INTERFACE}\necho
CASSANDRA_MEMTABLE_ALLOCATION_TYPE ${CASSANDRA_MEMTABLE_ALLOCATION_TYPE}\necho
CASSANDRA_MEMTABLE_CLEANUP_THRESHOLD ${CASSANDRA_MEMTABLE_CLEANUP_THRESHOLD}\necho
CASSANDRA_MEMTABLE_FLUSH_WRITERS ${CASSANDRA_MEMTABLE_FLUSH_WRITERS}\necho CASSANDRA_MIGRATION_WAIT
${CASSANDRA_MIGRATION_WAIT}\necho CASSANDRA_NUM_TOKENS ${CASSANDRA_NUM_TOKENS}\necho
CASSANDRA_RACK ${CASSANDRA_RACK}\necho CASSANDRA_RING_DELAY ${CASSANDRA_RING_DELAY}\necho
CASSANDRA_RPC_ADDRESS ${CASSANDRA_RPC_ADDRESS}\necho CASSANDRA_RPC_INTERFACE ${CASSANDRA_RPC_INTERFACE}\necho
CASSANDRA_SEEDS ${CASSANDRA_SEEDS}\necho CASSANDRA_SEED_PROVIDER ${CASSANDRA_SEED_PROVIDER}\n\n\n#
if DC and RACK are set, use GossipingPropertyFileSnitch\nif [[ $CASSANDRA_DC &&
$CASSANDRA_RACK ]]; then\n echo \"dc=$CASSANDRA_DC\" > $CASSANDRA_CONF_DIR/cassandra-rackdc.properties\n
\ echo \"rack=$CASSANDRA_RACK\" >> $CASSANDRA_CONF_DIR/cassandra-rackdc.properties\n
\ CASSANDRA_ENDPOINT_SNITCH=\"GossipingPropertyFileSnitch\"\nfi\n\nif [ -n \"$CASSANDRA_MAX_HEAP\"
]; then\n sed -ri \"s/^(#)?-Xmx[0-9]+.*/-Xmx$CASSANDRA_MAX_HEAP/\" \"$CASSANDRA_CONF_DIR/jvm.options\"\n
\ sed -ri \"s/^(#)?-Xms[0-9]+.*/-Xms$CASSANDRA_MAX_HEAP/\" \"$CASSANDRA_CONF_DIR/jvm.options\"\nfi\n\nif
[ -n \"$CASSANDRA_REPLACE_NODE\" ]; then\n echo \"-Dcassandra.replace_address=$CASSANDRA_REPLACE_NODE/\"
>> \"$CASSANDRA_CONF_DIR/jvm.options\"\nfi\n\nfor rackdc in dc rack; do\n var=\"CASSANDRA_${rackdc^^}\"\n
\ val=\"${!var}\"\n if [ \"$val\" ]; then\n\tsed -ri 's/^('\"$rackdc\"'=).*/\\1
'\"$val\"'/' \"$CASSANDRA_CONF_DIR/cassandra-rackdc.properties\"\n fi\ndone\n\n#
TODO what else needs to be modified\nfor yaml in \\\n broadcast_address \\\n
\ broadcast_rpc_address \\\n cluster_name \\\n disk_optimization_strategy \\\n
\ endpoint_snitch \\\n listen_address \\\n num_tokens \\\n rpc_address \\\n
\ start_rpc \\\n key_cache_size_in_mb \\\n concurrent_reads \\\n concurrent_writes
\\\n memtable_cleanup_threshold \\\n memtable_allocation_type \\\n memtable_flush_writers
\\\n concurrent_compactors \\\n compaction_throughput_mb_per_sec \\\n counter_cache_size_in_mb
\\\n internode_compression \\\n endpoint_snitch \\\n gc_warn_threshold_in_ms
\\\n listen_interface \\\n rpc_interface \\\n ; do\n var=\"CASSANDRA_${yaml^^}\"\n
\ val=\"${!var}\"\n if [ \"$val\" ]; then\n sed -ri 's/^(# )?('\"$yaml\"':).*/\\2
'\"$val\"'/' \"$CASSANDRA_CFG\"\n fi\ndone\n\necho \"auto_bootstrap: ${CASSANDRA_AUTO_BOOTSTRAP}\"
>> $CASSANDRA_CFG\n\n# set the seed to itself. This is only for the first pod,
otherwise\n# it will be able to get seeds from the seed provider\nif [[ $CASSANDRA_SEEDS
== 'false' ]]; then\n sed -ri 's/- seeds:.*/- seeds: \"'\"$POD_IP\"'\"/' $CASSANDRA_CFG\nelse
# if we have seeds set them. Probably StatefulSet\n sed -ri 's/- seeds:.*/-
seeds: \"'\"$CASSANDRA_SEEDS\"'\"/' $CASSANDRA_CFG\nfi\n\nsed -ri 's/- class_name:
SEED_PROVIDER/- class_name: '\"$CASSANDRA_SEED_PROVIDER\"'/' $CASSANDRA_CFG\n\n#
set read and write time out in yaml\nif [[ $write_request_timeout_in_ms ]]; then\n
\ sed -ri 's/write_request_timeout_in_ms:.*/write_request_timeout_in_ms: \"'\"$write_request_timeout_in_ms\"'\"
/' $CASSANDRA_CFG\nfi\n\nif [[ $read_request_timeout_in_ms ]]; then\n sed -ri
's/read_request_timeout_in_ms:.*/read_request_timeout_in_ms: \"'\"$read_request_timeout_in_ms\"'\"
/' $CASSANDRA_CFG\nfi\n\n# send gc to stdout\nif [[ $CASSANDRA_GC_STDOUT == 'true'
]]; then\n sed -ri 's/ -Xloggc:\\/var\\/log\\/cassandra\\/gc\\.log//' $CASSANDRA_CONF_DIR/cassandra-env.sh\nfi\n\n#
enable RMI and JMX to work on one port\necho \"JVM_OPTS=\\\"\\$JVM_OPTS -Djava.rmi.server.hostname=$POD_IP\\\"\"
>> $CASSANDRA_CONF_DIR/cassandra-env.sh\n\n# getting WARNING messages with Migration
Service\necho \"-Dcassandra.migration_task_wait_in_seconds=${CASSANDRA_MIGRATION_WAIT}\"
>> $CASSANDRA_CONF_DIR/jvm.options\necho \"-Dcassandra.ring_delay_ms=${CASSANDRA_RING_DELAY}\"
>> $CASSANDRA_CONF_DIR/jvm.options\n\n# ignore empty tables\necho \"-Dcassandra.ignore_corrupted_schema_tables=true\"
>> $CASSANDRA_CONF_DIR/jvm.options\n\nif [[ $CASSANDRA_OPEN_JMX == 'true' ]];
then\n export LOCAL_JMX=no\n sed -ri 's/ -Dcom\\.sun\\.management\\.jmxremote\\.authenticate=true/
-Dcom\\.sun\\.management\\.jmxremote\\.authenticate=false/' $CASSANDRA_CONF_DIR/cassandra-env.sh\n
\ sed -ri 's/ -Dcom\\.sun\\.management\\.jmxremote\\.password\\.file=\\/etc\\/cassandra\\/jmxremote\\.password//'
$CASSANDRA_CONF_DIR/cassandra-env.sh\nfi\n\nchmod 700 \"${CASSANDRA_DATA}\"\nchown
-c -R cassandra \"${CASSANDRA_DATA}\" \"${CASSANDRA_CONF_DIR}\"\n\nexport CLASSPATH=/kubernetes-cassandra.jar\n\nsu
cassandra -c \"$CASSANDRA_HOME/bin/cassandra -f\"\n"
kind: ConfigMap
metadata:
creationTimestamp: "2020-04-07T06:41:33Z"
name: cassandra-config
namespace: director
resourceVersion: "3019"
selfLink: /api/v1/namespaces/director/configmaps/cassandra-config
uid: d27afad0-789a-11ea-899a-42010a800092
This basically happens because of the time read and write timeout is set to very high(as shown in the above output in red).
Reduce these values to some lower values and restart the Cassandra pod.
Example values:
write_request_timeout_in_ms:-30
read_request_timeout_in_ms:-50