sqoop 和mysql相关操作

数据迁移工具,可以和RDBMS相互迁移数据

需要先将db driver copy to sqoop lib dir
注意:
    sqoop是以mapreduce的方式来运行任务的,用hdfs来存储数据的。所以依赖NameNode和ResourceManager,只要机器上配置了这两个就可以正常运行,程序运行时会自动读取环境变量. (更多…)

mysql常用解决方案

1.删除了mysql数据库user表记录,会导致用户不能正常登录。

此时将mysqld 以skip-grant-tables启动,忽略权限。
进入到mysql库中,执行以下脚本。
就可以用root/bill登录了
  1. /*
  2. Navicat MySQL Data Transfer
  3. Source Server : localhost
  4. Source Server Version : 50703
  5. Source Host : localhost:3306
  6. Source Database : mysql
  7. Target Server Type : MYSQL
  8. Target Server Version : 50703
  9. File Encoding : 65001
  10. Date: 2015-06-13 20:07:50
  11. */
  12. SET FOREIGN_KEY_CHECKS=0;
  13. -- ----------------------------
  14. -- Table structure for `user`
  15. -- ----------------------------
  16. DROP TABLE IF EXISTS `user`;
  17. CREATE TABLE `user` (
  18. `Host` char(60) COLLATE utf8_bin NOT NULL DEFAULT '',
  19. `User` char(16) COLLATE utf8_bin NOT NULL DEFAULT '',
  20. `Password` char(41) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT '',
  21. `Select_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  22. `Insert_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  23. `Update_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  24. `Delete_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  25. `Create_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  26. `Drop_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  27. `Reload_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  28. `Shutdown_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  29. `Process_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  30. `File_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  31. `Grant_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  32. `References_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  33. `Index_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  34. `Alter_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  35. `Show_db_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  36. `Super_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  37. `Create_tmp_table_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  38. `Lock_tables_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  39. `Execute_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  40. `Repl_slave_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  41. `Repl_client_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  42. `Create_view_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  43. `Show_view_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  44. `Create_routine_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  45. `Alter_routine_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  46. `Create_user_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  47. `Event_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  48. `Trigger_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  49. `Create_tablespace_priv` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  50. `ssl_type` enum('','ANY','X509','SPECIFIED') CHARACTER SET utf8 NOT NULL DEFAULT '',
  51. `ssl_cipher` blob NOT NULL,
  52. `x509_issuer` blob NOT NULL,
  53. `x509_subject` blob NOT NULL,
  54. `max_questions` int(11) unsigned NOT NULL DEFAULT '0',
  55. `max_updates` int(11) unsigned NOT NULL DEFAULT '0',
  56. `max_connections` int(11) unsigned NOT NULL DEFAULT '0',
  57. `max_user_connections` int(11) unsigned NOT NULL DEFAULT '0',
  58. `plugin` char(64) COLLATE utf8_bin NOT NULL DEFAULT 'mysql_native_password',
  59. `authentication_string` text COLLATE utf8_bin,
  60. `password_expired` enum('N','Y') CHARACTER SET utf8 NOT NULL DEFAULT 'N',
  61. PRIMARY KEY (`Host`,`User`)
  62. ) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='Users and global privileges';
  63. -- ----------------------------
  64. -- Records of user
  65. -- ----------------------------
  66. INSERT INTO `user` VALUES ('%', 'root', '*29A1BB43D3B9EB42028B4566E4836353285B9395', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', '', '', '', '', '0', '0', '0', '0', 'mysql_native_password', '', 'N');
  67. INSERT INTO `user` VALUES ('127.0.0.1', 'root', '*29A1BB43D3B9EB42028B4566E4836353285B9395', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', '', '', '', '', '0', '0', '0', '0', 'mysql_native_password', '', 'N');
  68. INSERT INTO `user` VALUES ('::1', 'root', '*29A1BB43D3B9EB42028B4566E4836353285B9395', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', 'Y', '', '', '', '', '0', '0', '0', '0', 'mysql_native_password', '', 'N');
  69. INSERT INTO `user` VALUES ('localhost', '', '', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', 'N', '', '', '', '', '0', '0', '0', '0', 'mysql_native_password', null, 'N');

(更多…)

hive UDF笔记

1.编写代码

  1. package com.billstudy.udf;
  2. import java.util.HashMap;
  3. import java.util.Map;
  4. import org.apache.hadoop.hive.ql.exec.UDF;
  5. import org.apache.hadoop.io.Text;
  6. /**
  7. * 自定义UDF函数
  8. * @author Bill
  9. * @since V1.0 2015年6月14日 - 下午10:33:00
  10. */
  11. public class NationUDF extends UDF{
  12. public static Map<String,String> nationMap = new HashMap<String,String>();
  13. static {
  14. nationMap.put("china", "中国");
  15. nationMap.put("japan", "日本");
  16. nationMap.put("usa", "美国");
  17. }
  18. private Text returnNation = new Text();
  19. /**
  20. * 名称必须是evaluate
  21. * @author Bill
  22. * @since V1.0 2015年6月14日 - 下午10:32:41
  23. * @param nationKey
  24. * @return
  25. */
  26. public Text evaluate(Text nationKey){
  27. String nation = nationMap.get(nationKey.toString());
  28. if (nation == null) {
  29. nation = "未知";
  30. }
  31. returnNation.set(nation);
  32. return returnNation;
  33. }
  34. }

(更多…)