For server Eagle-PROD-Instance, the Node Manager associated with machine

Question

I have a wlst script that creates a domain and a managed server. I associate the server with a machine that's attached to a node manager. It creates the managed server fine but when I try and start it I get this error
For server Eagle-PROD-Instance, the Node Manager associated with machine Eagle-Machine is not reachable.
All of the servers selected are currently in a state which is incompatible with this operation or are not associated with a running Node Manager or you are not authorized to perform the action requested. No action will be performed.
The machine is associated with the Node Manager and the NM is running. I can start the managed server from the command line, but not from the admin console
This is the script, am I missing something?
Thanks
name: createManagedServer.py
description: This script create the weblogic domain, and executes each weblogic queue module
subfile. it reads a property file : weblogic_wlst.properties for server domain information
author     : mike reynolds - edifecs 2011
created     : April 8th 2011
import sys
from java.lang import System
from java.util import Properties
from java.io import FileInputStream
from java.io import File
from weblogic.management.security.authentication import UserEditorMBean
from weblogic.management.security.authentication import GroupEditorMBean
from weblogic.management.security.authentication import UserPasswordEditorMBean
# Loads the contents a properties file into a java.util.Properties
# configPropFile = "weblogic_wlst.properties"
def loadPropertiesFromFile(configPropFile):
     configProps = Properties()
     propInputStream = FileInputStream(configPropFile)
     configProps.load(propInputStream)
     propInputStream.close()
     return configProps
def getProperties():
     importConfigFile = sys.argv[1]
     print importConfigFile
     domainProps = loadPropertiesFromFile(importConfigFile)
     properties = Properties()
     input = FileInputStream(importConfigFile)
     properties.load(input)
     input.close()
     return properties
def create_users(username, password, description):
# create admin user
cmo.getFileRealms()
try:
userObject=cmo.getSecurityConfiguration().getDefaultRealm().lookupAuthenticationProvider("DefaultAuthenticator")
userObject.createUser(username,password,description)
print "Created user " + username + "successfully"
except:
print "check to see if user " + username + " exists "
def add_user_to_group(username):
print "Adding a user to group ..."
cmo.getFileRealms()
try:
userObject2 = cmo.getSecurityConfiguration().getDefaultRealm().lookupAuthenticationProvider("DefaultAuthenticator")
userObject2.addMemberToGroup('Administrators',username)
print "Done adding user " + username
except:
print "check to see if user " + username + " is already in group "
def connect_server(user,pw,url):
          connect(user,pw,url)
def create_machine():     
try:
print 'Creating machine' + machine
# cd('/')
# create(machine, 'Machine')
mach = cmo.createUnixMachine(machine)
mach.setPostBindUIDEnabled(true)
mach.setPostBindUID('oracle')
mach.setPostBindGIDEnabled(true)
mach.setPostBindGID('oracle')
mach.getNodeManager().setNMType('ssl')
except:
     print "machine exists"
def build_domain():
### Read Basic Template
WL_HOME = "C:/Oracle/Middleware/wlserver_10.3"     
readTemplate(WL_HOME+"/common/templates/domains/wls.jar")
template=WL_HOME+"/common/templates/domains/wls.jar"
cd('Servers/AdminServer')
set('ListenAddress', adminServerAddress)
set('ListenPort', int(adminServerPort))
cd('/')
cd('/Security/base_domain/User/weblogic')
cmo.setPassword('w3bl0g1c')
### Write Domain
setOption('OverwriteDomain', 'true')
print "writing domain " + domainDir + domainName
writeDomain(domainDir+'/'+domainName)
closeTemplate()
create_machine()
arg = "Arguments=\" -server -Xms256m -Xmx768m -XX:MaxPermSize=256m -da\""
prps = makePropertiesObject (arg)
domain = domainDir + domainName
try:
     #startNodeManager()
#nmConnect('weblogic', 'w3bl0g1c', host, 5556, 'AdminServer', domain, 'ssl')
# nmStart('AdminServer')
startServer('AdminServer', domainName, url, adminUser, adminPassword, domainDir, 'true')
except:
print "could not connect to Node Manager"
def create_server():     
# get server instance properties
name = properties.getProperty("serverName")
domain = properties.getProperty("domainName")
port = properties.getProperty("listenPort")
address = properties.getProperty("listenAddress")
servermb=getMBean("Servers/" + name)
machine = properties.getProperty("machineName")
nodePort = properties.getProperty("nodeManagerPort")
domainDir = properties.getProperty("domainDir")
if servermb is None:
          startEdit()
          cd('/')
          cmo.createServer(name)
          cd('/Servers/'+ name)
          cmo.setListenAddress(address)
          cmo.setListenPort(int(port))
          cd('/')
          cmo.createMachine(machine)
          cd('/Machines/' + machine + '/NodeManager/' + machine )
          cmo.setNMType('Plain')
          cmo.setListenAddress(address)
          cmo.setListenPort(int(nodePort))
          cmo.setDebugEnabled(false)
          cd('/Servers/' + name)
          cmo.setListenPortEnabled(true)
          cmo.setJavaCompiler('javac')
          cmo.setClientCertProxyEnabled(false)
          cmo.setMachine(getMBean('/Machines/' + machine ))
          cmo.setCluster(None)
          cd('/Servers/' + name + '/SSL/' + name)
          cd('/Servers/' + name + '/ServerDiagnosticConfig/' + name)
          cmo.setWLDFDiagnosticVolume('Low')
          cd('/Servers/' + name)
          cmo.setCluster(None)
          cd('/Servers/' + name + '/SSL/' + name)
          cmo.setEnabled(false)
### Executable Script
### CreateDomain.py
### Define constants
WL_HOME = "C:/Oracle/Middleware/wlserver_10.3"
print "Starting the script ..."
print "Getting properties ... "
properties = getProperties()
adminServerAddress = properties.getProperty("adminServerAddress")
adminServerPort = properties.getProperty("adminServerPort")
adminUser = properties.getProperty("adminUser")
adminPassword = properties.getProperty("adminPassword")
edifecsUser = properties.getProperty("edifecsUser")
edifecsPassword = properties.getProperty("edifecsPassword")
host = properties.getProperty("host")
domainDir = properties.getProperty("domainDir")
domainName = properties.getProperty("domainName")
user = properties.getProperty("username")
pw = properties.getProperty("passwd")
url = properties.getProperty("adminURL")
machine = properties.getProperty("machineName")
print "Building the domain..."
build_domain()
print "Connecting to server"     
connect_server(adminUser, adminPassword, url)
edit()
startEdit()
# create managed server
# create_machine()
create_server()
print "Creating users"
# starting configuration tree
serverConfig()
create_users(adminUser, adminPassword, "Administrator")
add_user_to_group(adminUser)
create_users(edifecsUser, edifecsPassword,"Administrator")
add_user_to_group(edifecsUser)
# have to restart edit to save config
edit()
startEdit()
# nmKill('AdminServer')
print "saving configuration"
try:
     save()
     activate(block="true")
     print "script returns SUCCESS"
     print "admin server is running"
     print "starting server " + name
     startServer(domainName, name ,url,adminUser, adminPassword, domainDir,'true')
except:
     print "failed to save server"
     dumpStack()

Answer 1

Actually the cert is coming from your Dev machine but it is sending the Prod cert.
What cert is used by your admin server ? It should match the host name.
So your Dev machine is apparently using a copy of the prod cert / keystore rather than using its own DEV cert. It's not clear from your post whether this is the nodemanager using the wrong cert, or the managed server. So both should be checked.
The managed servers need to be using a cert that matches their host name. If you have a managed server on VM-BEA-DEV, then the cert needs to be CN=VM-BEA-DEV. You can also use a load-balancer CN name in the cert if you have the cluster's HTTP values set to match.
In your nodemanager.properties, are you explicitly accessing keystores, such as with:
KeyStores=CustomIdentityAndJavaStandardTrust
CustomIdentityAlias=some_alias
CustomIdentityKeyStoreFileName=some_path_to_keystore
CustomIdentityKeyStorePassPhrase={3DES}...
CustomIdentityKeyStoreType=jks
CustomIdentityPrivateKeyPassPhrase={3DES}
In my multi-machine clusters, I have multiple certificates such as:
admin machine1:
has a cert for use by the admin server and NM that matches the host name ( with node manager.properties entries such as the above )
has a 2nd cert that matches the load-balancer name for the cluster - used by the managed servers
all other machines:
has a cert for use by NM that matches the host name ( with node manager.properties entries such as the above )
has a 2nd cert that matches the load-balancer name for the cluster - used by the managed servers