hadoop组映射从配置文件读取

hadoop实现类似linux系统的文件权限,需要知道某个用户是属于哪个组。系统默认是使用ShellBasedUnixGroupsMapping 类来获取某个用户属于哪个组,实际是调用id命令来获取用户组信息。为了方便管理,也可以自己实现这个类,通过读取文件的方式来获得组信息。只要实现org.apache.hadoop.security.GroupMappingServiceProvider接口即可,具体可以参考ShellBasedUnixGroupsMapping的实现。

 

代码

package cn.uc.hadoop.security;
 
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.GroupMappingServiceProvider;
 
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
 
/**
 * Created by fatkun on 2016/7/31.
 */
public class FileBasedUnixGroupsMapping implements GroupMappingServiceProvider {
    private ConcurrentHashMap<String, List<String>> groups = null;
    private static final Log LOG = LogFactory.getLog(FileBasedUnixGroupsMapping.class);
    private String confFilePath = "/home/cloudera/var/group/groups.txt";
    // 格式: user1=group1,group2
 
    public void setConfFilePath(String confFilePath) {
        this.confFilePath = confFilePath;
    }
 
    private synchronized void loadConf(boolean refresh) {
 
        if (refresh || groups == null) {
            ConcurrentHashMap<String, List<String>> tmpGroups = new ConcurrentHashMap<>();
 
            try {
                BufferedReader reader = new BufferedReader(new FileReader(confFilePath));
                String line = null;
                while ((line = reader.readLine()) != null) {
                    line = line.trim();
                    if (line.startsWith("#")) continue;
                    String[] ugArray = line.split("=");
                    if (ugArray.length == 2) {
                        String user = ugArray[0].trim();
                        List<String> groupList = new ArrayList<>();
                        Set<String> groupSet = new HashSet<String>();
                        groupSet.add(user);
                        for (String group : ugArray[1].split(",")) {
                            groupSet.add(group.trim());
                        }
                        groupList.addAll(groupSet);
                        LOG.info("add user=" + user + " groups=" + groupList);
                        tmpGroups.put(user, groupList);
                    }
                }
                reader.close();
            } catch (IOException e) {
                LOG.error("LOAD GROUP MAPPING ERROR", e);
            }
            groups = tmpGroups;
        }
    }
 
    @Override
    public List<String> getGroups(String user) {
        if (groups == null) {
            loadConf(false);
        }
 
        List<String> groupList = groups.get(user);
 
        // 如果找不到组配置,返回一个和用户名一样的组名称
        if (groupList == null) {
            groupList = new ArrayList<>();
            groupList.add(user);
            LOG.error("CAN'T NOT FIND GROUPS FOR USER " + user);
        }
 
        return groupList;
    }
 
    @Override
    public void cacheGroupsRefresh() {
        loadConf(true);
    }
 
    @Override
    public void cacheGroupsAdd(List<String> groups) throws IOException {
        // does nothing in this provider of user to groups mapping
    }
 
    public static void main(String[] args) throws IOException {
        FileBasedUnixGroupsMapping f = new FileBasedUnixGroupsMapping();
        f.setConfFilePath("d:/tmp/groups.txt");
        System.out.println(f.getGroups("impala"));
    }
}

 

在cloudera如何使用

在namenode的配置“hdfs-site.xml 的 NameNode 高级配置代码段”加入以下配置,注意不要改“服务级别的hadoop.security.group.mapping”,我们只打算改掉namenode获取组映射的方式

  <property>
    <name>hadoop.security.group.mapping</name>
    <value>cn.uc.hadoop.security.FileBasedUnixGroupsMapping</value>
  </property>

在 “NameNode 环境高级配置代码段(安全阀)”加入(把jar加入namenode的classpath里面)

HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/home/cloudera/var/group/hadoopgroup.jar

生成groups.txt见 http://fatkun.com/2016/08/get-user-group-mapping.html




fatkun

折腾一下又不会死~

一条评论

肥坤同志写的软件都挺好用的,哪天搞不好你就成为下一个张小龙了

发表评论

电子邮件地址不会被公开。