Malloc fails after 3.8 GB when jvm is loaded on SunOS 11

Question

All,
I have a very simple c++ program that dynamically loads the jvm and in a loop tries to allocate memory in chunks of 512K. The malloc fails after about 3.8 GB is consumed. I don't use the jvm in any way other than just loading it. If I specify a java max heap size of 4096 MB I can allocate up 29 GB and then the malloc fails. If I subsequently increase the max heap size to 32GB then I can allocate more than 100GB.
What is interesting is that this happens only on SunOS 11. Also it happens with java 1.6.0_26 and java 1.7 but not with java 1.6.0_22. If I don't load the jvm everything runs fine. This simple program runs without issues on SunOS 10, linux and windows with java 1.7.
I have used truss with this program and I get the following output
/1: nanosleep(0xFFFFFD7FFFDFF600, 0x00000000) = 0
/1: brk(0xEAAEC000) = 0
/1: nanosleep(0xFFFFFD7FFFDFF600, 0x00000000) = 0
/1: brk(0xEABED000) = 0
/1: nanosleep(0xFFFFFD7FFFDFF600, 0x00000000) = 0
/1: brk(0xEACEE000) Err#12 ENOMEM
This machine has 256GB of memory. I have been scratching my head for a while now and any help will be greatly appreciated.
Pauli
Edited by: 948514 on Jul 24, 2012 3:01 PM

Answer 1

Find below the complete code that is encountering this issue
#define MAX_MEMORY 10*1024*1024
#define ALLOC_SIZE 512
#define MAX_JAVA_HEAP "-Xmx256m"
#define REPORTING_INTERVAL 1000
#define USE_JAVA_VM 1
#ifdef _WIN32
#include <winsock2.h>
#include "win_dlfcn.h"
#else
#include <dlfcn.h>
#endif
#include <jni.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <string>
#include <iostream>
#include <list>
using namespace std;
//On Windows the JNI_CreateJavaVM function is called with __stdcall convention
#ifdef _WIN32
typedef jint (__stdcall jniCreateJavaVM_t) (JavaVM *, void **, JavaVMInitArgs *);
typedef jint (__stdcall jniGetCreatedJavaVMs_t) (JavaVM *, jsize, jsize *);
#else
typedef jint (*jniCreateJavaVM_t) (JavaVM **, void **, JavaVMInitArgs *);
typedef jint (*jniGetCreatedJavaVMs_t) (JavaVM **, jsize, jsize *);
#endif
static void *s_jvmLibrary = NULL;
static JavaVMInitArgs s_vmArgs = {0};
static JavaVMOption *s_vmOptions = NULL;
string maxHeapSize = "-XmxMAX_JAVA_HEAP";
void *
GetJavaVMLibrary()
std::string jvmLibrary;
const char *pjvmLibrary = getenv("JAVA_HOME");
if (!pjvmLibrary || !*pjvmLibrary)
std::cerr << "JAVA_HOME has not been set" << std::endl;
exit(1);
jvmLibrary = std::string(pjvmLibrary) + "/";
#if defined(WIN64)
          jvmLibrary.append("bin/server/jvm.dll");
#elif defined(WIN32)
          jvmLibrary.append("bin/client/jvm.dll");
#elif defined(__sparc__) && defined(__sun__)
          jvmLibrary.append("lib/sparcv9/server/libjvm.so");
#elif defined(__x86_64__) && defined(__sun__)
          jvmLibrary.append("lib/amd64/server/libjvm.so");
#elif defined(__i386__) && defined(__linux__)
          jvmLibrary.append("lib/i386/server/libjvm.so");
#elif defined(__x86_64__) && defined(__linux__)
          jvmLibrary.append("lib/amd64/server/libjvm.so");
#else
          jvmLibrary = "./libjvm.so";
#endif
std::cout << "jvmlibrary is " << jvmLibrary << std::endl;
#ifndef RTLD_GLOBAL
#define RTLD_GLOBAL 0
#endif
void * s_jvmLibrary = dlopen(jvmLibrary.c_str(), RTLD_LAZY | RTLD_GLOBAL);
if( !s_jvmLibrary )
char * err = dlerror();
string error = (err == NULL) ? "could not open Java VM shared library" : err;
std::cerr << error << std::endl;
exit(1);
     return s_jvmLibrary;
JavaVM *
CreateJavaVM()
     JNIEnv *env = NULL;
     JavaVM *jvm = NULL;
     std::list<std::string> vmOptions;
     void *libraryHandle = GetJavaVMLibrary();
     jniCreateJavaVM_t createVMFunction = (jniCreateJavaVM_t)dlsym(libraryHandle, "JNI_CreateJavaVM");
     if( !createVMFunction )
          std::cerr << "could not find Java VM library function " << std::endl;
exit(1);
//Set the Java Max Heap Option
int noOfOptions = 1;
     s_vmOptions = (JavaVMOption *)malloc(noOfOptions * sizeof(JavaVMOption));
     s_vmOptions[0].optionString = strdup(maxHeapSize.c_str());
     s_vmArgs.version = JNI_VERSION_1_6;
     s_vmArgs.options = s_vmOptions;
     s_vmArgs.nOptions = noOfOptions;
     s_vmArgs.ignoreUnrecognized = JNI_TRUE;
     // Create the Java VM
     int ret = createVMFunction(&jvm, (void**)&env, &s_vmArgs);
     if( ret != 0 )
std::cerr << "Could not create jvm" << endl;
exit(1);
     return jvm;
int
main(int argc, char **argv)
int createJvm = USE_JAVA_VM;
long mSize = ALLOC_SIZE;
long maxSize = MAX_MEMORY;
if(argc > 1){
maxSize = (atol(argv[1]) * 1024);
cout << "maxSize is " << maxSize << endl;
if(argc > 2){
maxHeapSize = string("-Xmx").append(argv[2]);
if(argc > 3){
createJvm = atoi(argv[3]);
     if (createJvm)
          JavaVM *vm = CreateJavaVM();
          if (vm != NULL)
               printf("Sucessfully created Java VM\n");
          else
               printf("Failed to create Java VM\n");
               return 1;
     long memUsed = 0;
     long count = 0;
     printf("beginning mallocs\n");
     std::list<void *> memory;
     while (memUsed < maxSize)
          void ptr = malloc(mSize1024);
          memory.push_back(ptr);
          if (ptr == NULL)
               printf("malloc failed, count=%ld, size=%ld k\n", count, memUsed);
               return 1;
          memset(ptr, 0, mSize*1024);
          memUsed += mSize;
          count++;
          if (!(count % REPORTING_INTERVAL))
          printf("malloc count=%ld, size=%ld m\n", count, memUsed/1024);
     printf("finished mallocs\n");
     return 0;
}