Published on

Hadoop-Client NoClassDefFoundError 발생 해결 방법(intellij)

Authors
  • Name
    Twitter

Overview

Hadoop 클러스터에 접근하는 Java 프로그램을 수행하는데, 아래와 같은 애러가 발생한다.

Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/hadoop/conf/Configuration
	at org.example.Main.main(Main.java:18)
Caused by: java.lang.ClassNotFoundException: org.apache.hadoop.conf.Configuration
	at java.net.URLClassLoader.findClass(URLClassLoader.java:387)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:418)
	at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352)
	at java.lang.ClassLoader.loadClass(ClassLoader.java:351)
	... 1 more

Main.java
package org.example;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;

public class Main {

    public static void main(String[] args) throws IOException {
        Configuration configuration = new Configuration();
        configuration.set("fs.defaultFS", "hdfs://my_hdfs_fqdn:8020");
        FileSystem fileSystem = FileSystem.get(configuration);
        String directoryName = "tmp/test";
        Path path = new Path(directoryName);
        fileSystem.mkdirs(path);

    }
}

해결 방법

dependency에서 scope를 주석처리 해주면, 클래스를 찾지 못하는 애러를 해결할 수 있다.

pom.xml
    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.6.0</version>
            <!-- <scope>provided</scope> -->
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>2.6.0</version>
        </dependency>

    </dependencies>