Used splunk provided directions on the following page to configure:
http://wiki.splunk.com/Community:Splunk2Splunk_SSL_3rdPartyCA
Configuring Splunk forwarding to use SSL certificates signed by a third party Certificate Authority does not work
a. When using third party certificates (Microsoft CA Server), Splunk fails to forward data to the Indexer.
b. When Splunk is configured to use the built-in self-signed certificates, Splunk Forwarding works with no problem.
c. Shown below is the broken configuration.
d. The only difference between the broken and working configurations is the certificates. The working configuration uses the default Splunk self-signed certificates in:
i. On the Indexer: /opt/splunk/etc/auth/server.pem and cacert.pem
ii. On the Forwarder: /opt/splunkforwarder/etc/auth/server.pem and cacert.pem
Configuration and Certs On Indexer:
2. /opt/splunk/etc/system/local/inputs.conf
[default]
index = default
_rcvbuf = 1572864
host = $decideOnStartup
[monitor://$SPLUNK_HOME/var/log/splunk]
index = _internal
[monitor://$SPLUNK_HOME/etc/splunk.version]
_TCP_ROUTING = *
index = _internal
sourcetype = splunk_version
[batch://$SPLUNK_HOME/var/spool/splunk]
move_policy = sinkhole
crcSalt =
[batch://$SPLUNK_HOME/var/spool/splunk/...stash_new]
queue = stashparsing
sourcetype = stash_new
move_policy = sinkhole
crcSalt =
[fschange:$SPLUNK_HOME/etc]
pollPeriod = 600
signedaudit = true
recurse = true
followLinks = false
hashMaxSize = -1
fullEvent = false
sendEventMaxSize = -1
filesPerDelay = 10
delayInMills = 100
[udp]
connection_host = ip
[tcp]
acceptFrom = *
connection_host = dns
[splunktcp]
route = has_key:_replicationBucketUUID:replicationQueue;has_key:_dstrx:typingQueue;has_key:_linebreaker:indexQueue;absent_key:
_linebreaker:parsingQueue
acceptFrom = *
connection_host = ip
[script]
interval = 60.0
[splunktcp-ssl:9997]
compressed = true
[splunktcp://9997]
connection_host = none
[SSL]
password = $1$d9nAgrJsGkWc
rootCA = /opt/splunk/etc/auth/mycerts/mycacert.pem
serverCert = /opt/splunk/etc/auth/mycerts/myServerCertificate.pem
openssl rsa -in /opt/splunk/etc/auth/mycerts/myServerCertificate.pem -text
Enter pass phrase for /opt/splunk/etc/auth/mycerts/myServerCertificate.pem:
Private-Key: (1024 bit)
modulus:
00:c5:ed:76:43:11:14:25:7e:32:20:19:7c:30:f0:
ba:45:9a:74:65:28:a3:26:52:32:d0:6b:b0:0d:6c:
df:57:d3:6e:e2:a3:8d:e6:ae:4e:97:8f:a8:be:81:
f4:97:88:60:6f:35:44:83:48:63:b2:73:60:99:31:
25:63:2d:c6:d4:6a:8e:a7:52:01:8f:72:6e:f5:e6:
51:b2:e1:2c:01:1e:da:13:d3:eb:16:80:00:1d:d8:
87:40:9a:62:c6:f8:72:3b:21:a8:05:e3:ba:c5:c4:
04:6b:85:4c:d3:dd:0f:d8:75:a3:b3:7f:a8:2e:a9:
14:00:20:84:e3:9a:c5:fa:27
...
writing RSA key
-----BEGIN RSA PRIVATE KEY-----
MIICXQIBAAKBgQDF7XZDERQlfjIgGXww8LpFmnRlKKMmUjLQa7ANbN9X027io43m
rk6Xj6i+gfSXiGBvNUSDSGOyc2CZMSVjLcbUao6nUgGPcm715lGy4SwBHtoT0+sW
gAAd2IdAmmLG+HI7IagF47rFxARrhUzT3Q/YdaOzf6guqRQAIITjmsX6JwIDAQAB
AoGBALMOF6aklK02dPJFG+zKWjkNea7qDG5mfkG+qg37KDGzvOSbQYwmtEK4W9e8
iSFs5pC0h76chlSxu/naVBBdITj/0pv0hwH/p+1lvNNSqBAQ3ROOok7yInvidg1F
BUo9chELxX7Yp+X6Fs5IW9RgNI5mSKTKdezJESu81A7Qa7xBAkEA+DxouEnnmz8h
tkY10+Im7AbXEVRwZzxnkU0Ikr7YIIs1tpnznHZuasGGXoYoYG1PeeM6fgKUDKPp
p+ymGAhC7QJBAMwePZo5BsVsXIFidruUPyoZGWgGecsJOLoKclww8ROtnebCuKWK
eEtasZ3WZrGexqF+ld8F2D2XRgu3GzCe6uMCQQCxx9HX6lYNQXGLcU0rqlPlxiBR
MQAvb3tc/KafMj7nT8vwMuHdtJPvsRniqIJSTPcWfD5v8LjHNL0qnrl1jLUhAkBz
/ScyUP95BjeWylYAB6DREkwuoadp6caTaUZM/v6vGPRmYfY9E2+CGnpd36yheEEV
GfKeNhsH/MMv+w/3VAbTAkBxgiOgVsMjV8GpKY6YA9mKaowCPTGaeY/9uwXbALvI
XNAURK5Da0TNKBOwNjJ9Ti8ZPai5CE7dGsZQTHh97DEx
-----END RSA PRIVATE KEY-----
openssl x509 -in /opt/splunk/etc/auth/mycerts/myServerCertificate.pem -text -noout
Certificate:
Data:
Version: 3 (0x2)
Serial Number:
16:a7:28:0e:00:00:00:00:00:6d
Signature Algorithm: sha1WithRSAEncryption
Issuer: DC=gov, DC=ic, DC=army, DC=infra, CN=INFRADC1
Validity
Not Before: Jul 23 17:29:02 2013 GMT
Not After : Jul 23 17:29:02 2015 GMT
Subject: C=US, ST=VA, L=Springfield, O=GSS-CGI, OU=DCAC, CN=eas1.infra.army.ic.gov
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
RSA Public Key: (1024 bit)
Modulus (1024 bit):
00:b7:51:b1:1f:af:ed:c5:1a:d6:b0:16:6b:c4:1c:
b9:6f:65:84:79:2f:7e:db:11:35:7b:a6:a3:2c:2e:
0c:eb:39:c0:b0:81:03:88:78:07:6a:46:9c:04:25:
46:ef:6d:41:88:e1:18:4f:ae:2b:30:bb:7e:9d:7d:
23:d9:8c:c3:2d:17:41:02:9e:a8:17:d7:08:0c:9e:
68:cd:c5:af:2e:51:2e:9f:ef:62:a5:56:79:a0:e0:
c3:c4:92:3e:90:ac:e9:da:bc:8c:41:e3:37:aa:08:
bc:de:92:8e:b7:5f:49:da:eb:e8:5a:fa:af:d4:8b:
eb:df:c8:d8:ed:98:07:31:87
Exponent: 65537 (0x10001)
...
tail -f /opt/splunk/var/log/splunk/splunkd.log
...
07-29-2013 13:08:32.604 -0400 DEBUG TcpInputProc - Successfully negotiated capability with V3 protocol. Caps=ack=0;compression=0
...
Configuration and Certs On the Forwarder:
cat /opt/splunkforwarder/etc/system/local/outputs.conf
Version 5.0.3
[tcpout]
defaultGroup = splunkssl
maxQueueSize = 500KB
forwardedindex.0.whitelist = .*
forwardedindex.1.blacklist = _.*
forwardedindex.2.whitelist = (_audit|_internal)
forwardedindex.filter.disable = false
indexAndForward = false
autoLBFrequency = 30
blockOnCloning = true
compressed = false
disabled = false
dropClonedEventsOnQueueFull = 5
dropEventsOnQueueFull = -1
heartbeatFrequency = 30
maxFailuresPerInterval = 2
secsInFailureInterval = 1
maxConnectionsPerIndexer = 2
forceTimebasedAutoLB = false
sendCookedData = true
connectionTimeout = 20
readTimeout = 300
writeTimeout = 300
useACK = false
[tcpout:splunkssl]
compressed = true
server = 10.20.100.15:9997
sslCertPath = /opt/splunkforwarder/etc/auth/mycerts/myServerCertificate.pem
sslPassword = $1$w6IdRdDtFjxG
sslRootCAPath = /opt/splunkforwarder/etc/auth/mycerts/mycacert.pem
sslVerifyServerCert = false
openssl rsa -in myServerCertificate.pem -text
Enter pass phrase for myServerCertificate.pem:
Private-Key: (1024 bit)
modulus:
00:9d:87:c5:b2:e7:d2:ea:72:09:12:21:3f:5a:16:
c7:33:4f:b8:ae:0f:0b:62:78:2a:1b:e2:66:6b:b3:
3e:20:5b:3d:80:c4:d2:b0:c2:4d:43:d8:37:2b:2f:
13:7f:1b:19:4e:9b:90:76:85:6e:62:5b:52:41:b9:
e7:42:dc:b3:bd:95:da:7a:1d:f6:77:00:97:b1:14:
61:d4:a9:45:83:23:ea:24:09:ad:72:2b:62:65:60:
b7:73:e8:02:23:0e:b7:37:d8:1d:d2:a2:01:16:f8:
ef:96:bd:38:d5:47:9f:cb:a3:9c:c8:89:5d:42:cd:
da:df:8a:80:11:a8:3f:3e:49
publicExponent: 65537 (0x10001)
...
writing RSA key
-----BEGIN RSA PRIVATE KEY-----
MIICXAIBAAKBgQCdh8Wy59LqcgkSIT9aFsczT7iuDwtieCob4mZrsz4gWz2AxNKw
wk1D2DcrLxN/GxlOm5B2hW5iW1JBuedC3LO9ldp6HfZ3AJexFGHUqUWDI+okCa1y
K2JlYLdz6AIjDrc32B3SogEW+O+WvTjVR5/Lo5zIiV1CzdrfioARqD8+SQIDAQAB
AoGAXqDYmYe4oyytVj6yl6NnNeOFxMk0xYn5gZaWf8vEXhtw7pFNHvEZCNAxE7fL
tmbI5Pd96DRvApZo6yKJURjSvvak+HYjqTdLvCEN7yvPuh0IyAC9p2fq/uZplmsA
Sfd/bRfp3hWpUtQLzQN4m+PML/mrFbD86RedyRyUuONIGoECQQDLT5Guq8xzOK8j
2XwVTKrxyTgIhzqx46TpcKIZcneBB7auCUO2mOzzCe7oHybn4oB9a/DdPRqtCKyK
Fp/bDLY1AkEAxlrzFDn4q4L8B6tAUw5KglPe4pNbl/bg0H25K7BcELhEMu5IfCLD
gUACxYJafGcsNccIs9wmicG0Gs0VQSXaRQJAQXguAYFxJOlr/K9cNb+qjJGvaY+i
ZwZXZJTQnkEuGm7RdNmm5HX6V4krVbQyYxmdJsZLmfLDVFUmupDuiStewQJAdMRN
nHaUAMNXAly5vSsIibg92TvOC6N1rMaWHzXuvJj87M6BNTJxzMCV4RdflSRXTkEg
ymCq/yVclPptrLBP0QJBAKusMh/X28/QwAsgQrLOEhEjgwyVB0T8Si3s0jJBCaAB
gXPo663OGzhlQDoz4U+lLQzBqTS1nFY9B9E4RMaKLLM=
-----END RSA PRIVATE KEY-----
openssl x509 -in myServerCertificate.pem -text -noout
Certificate:
Data:
Version: 3 (0x2)
Serial Number:
15:70:b7:ff:00:00:00:00:00:76
Signature Algorithm: sha1WithRSAEncryption
Issuer: DC=gov, DC=ic, DC=army, DC=infra, CN=INFRADC1
Validity
Not Before: Jul 26 14:23:52 2013 GMT
Not After : Jul 26 14:23:52 2015 GMT
Subject: C=US, ST=VA, L=Springfield, O=GSS, OU=DCAC, CN=belv14dcacing\x1B
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
RSA Public Key: (1024 bit)
Modulus (1024 bit):
00:9d:87:c5:b2:e7:d2:ea:72:09:12:21:3f:5a:16:
c7:33:4f:b8:ae:0f:0b:62:78:2a:1b:e2:66:6b:b3:
3e:20:5b:3d:80:c4:d2:b0:c2:4d:43:d8:37:2b:2f:
13:7f:1b:19:4e:9b:90:76:85:6e:62:5b:52:41:b9:
e7:42:dc:b3:bd:95:da:7a:1d:f6:77:00:97:b1:14:
61:d4:a9:45:83:23:ea:24:09:ad:72:2b:62:65:60:
b7:73:e8:02:23:0e:b7:37:d8:1d:d2:a2:01:16:f8:
ef:96:bd:38:d5:47:9f:cb:a3:9c:c8:89:5d:42:cd:
da:df:8a:80:11:a8:3f:3e:49
Exponent: 65537 (0x10001)
...
tail -f /opt/splunkforwarder/var/log/splunk/splunkd.log
...
07-29-2013 13:08:30.268 -0400 DEBUG TcpOutputProc - Connection not available. Waiting for connection ...
07-29-2013 13:08:30.517 -0400 DEBUG TcpOutputProc - AutoLB timer started to select new connection
07-29-2013 13:08:30.517 -0400 DEBUG TcpOutputProc - Validating URI - 10.20.100.15:9997
07-29-2013 13:08:30.517 -0400 DEBUG TcpOutputProc - Validation complete
07-29-2013 13:08:30.517 -0400 DEBUG TcpOutputProc - Found host:10.20.100.15, port:9997 for DNS name :10.20.100.15:9997
07-29-2013 13:08:30.517 -0400 DEBUG TcpOutputProc - BEGIN - randomizeConnectionsList
07-29-2013 13:08:30.517 -0400 DEBUG TcpOutputProc - Indexer uri 10.20.100.15:9997, client refCount=0, client=NULL
07-29-2013 13:08:30.517 -0400 DEBUG TcpOutputProc - BEGIN - After sorting
07-29-2013 13:08:30.517 -0400 DEBUG TcpOutputProc - Indexer uri 10.20.100.15:9997, client refCount=0, client=NULL
07-29-2013 13:08:30.517 -0400 DEBUG TcpOutputProc - Found a candidate indexer which is currently not connected. 10.20.100.15:9997, client refCount=0, client=NULL
07-29-2013 13:08:30.517 -0400 DEBUG TcpOutputProc - Connection not available. Waiting for connection ...
07-29-2013 13:08:30.517 -0400 DEBUG TcpOutputProc - numchannels = 6
07-29-2013 13:08:30.517 -0400 DEBUG TcpOutputProc - ---- existing clients - start ----
07-29-2013 13:08:30.517 -0400 DEBUG TcpOutputProc - ---- existing clients - end ----
07-29-2013 13:08:30.517 -0400 DEBUG TcpOutputProc - Connector::runCookedStateMachine in state=eInit for 10.20.100.15:9997
07-29-2013 13:08:30.517 -0400 DEBUG TcpOutputProc - tcpConnect to 10.20.100.15:9997
...
... View more