diff --git a/conf/springConfigXml/sugonSdnController.xml b/conf/springConfigXml/sugonSdnController.xml index dd763b641ecf6087cf0542e1e7a50e675e4d5867..6188abb0a962b52218ab0dcd7fb88568f7573a61 100644 --- a/conf/springConfigXml/sugonSdnController.xml +++ b/conf/springConfigXml/sugonSdnController.xml @@ -71,4 +71,11 @@ + + + + + + + diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java index 9c1efd21e9a83b8f1919bb4308c3ec0b3cf6a9a6..37ac9763e670b7ef18a189aa6d71dade6b0b03cb 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java @@ -2952,7 +2952,7 @@ public class KVMHost extends HostBase implements Host { to.setIps(getCleanTrafficIp(nic)); } - if (!nic.getType().equals(VmInstanceConstant.VIRTUAL_NIC_TYPE)) { + if (!nic.getType().equals(VmInstanceConstant.VIRTUAL_NIC_TYPE) && !nic.getType().equals(VmInstanceConstant.TF_VIRTUAL_NIC_TYPE)) { return to; } diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java index 8cb1602ebaf3b97b7614687c70e4056b17b51d63..f8c66d4cd41cd357f585d6a261eb2c3fa7f58440 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/SugonSdnController.java @@ -16,9 +16,7 @@ import org.zstack.network.l2.vxlan.vxlanNetwork.L2VxlanNetworkInventory; import org.zstack.network.l3.L3NetworkSystemTags; import org.zstack.sdnController.SdnController; import org.zstack.sdnController.header.*; -import org.zstack.sugonSdnController.controller.api.ApiConnector; -import org.zstack.sugonSdnController.controller.api.ApiConnectorFactory; -import org.zstack.sugonSdnController.controller.api.Status; +import org.zstack.sugonSdnController.controller.api.*; import org.zstack.sugonSdnController.controller.api.types.*; import org.zstack.sugonSdnController.header.APICreateL2TfNetworkMsg; import org.zstack.utils.StringDSL; @@ -26,7 +24,6 @@ import org.zstack.utils.Utils; import org.zstack.utils.logging.CLogger; import org.zstack.utils.network.NetworkUtils; -import java.io.IOException; import java.util.*; import static org.zstack.core.Platform.operr; @@ -38,8 +35,11 @@ public class SugonSdnController implements TfSdnController, SdnController { private SdnControllerVO sdnControllerVO; + private TfHttpClient client; + public SugonSdnController(SdnControllerVO vo) { sdnControllerVO = vo; + client = new TfHttpClient(vo.getIp()); } @Override @@ -56,21 +56,20 @@ public class SugonSdnController implements TfSdnController, SdnController { return; } String accountUuid = StringDSL.transToTfUuid(accountVO.getUuid()); - ApiConnector apiConnector = ApiConnectorFactory.build(msg.getIp(), SugonSdnControllerGlobalProperty.TF_CONTROLLER_PORT); - assert apiConnector != null; - Domain domain = (Domain) apiConnector.findByFQN(Domain.class, SugonSdnControllerConstant.TF_DEFAULT_DOMAIN); + client = new TfHttpClient(msg.getIp()); + Domain domain = (Domain) client.getDomain(); if(domain == null){ completion.fail(operr("get default domain on tf controller failed")); return; } - Project defaultProject = (Project) apiConnector.findById(Project.class, accountUuid); + Project defaultProject = (Project) client.findById(Project.class, accountUuid); if(defaultProject == null){ Project project = new Project(); project.setParent(domain); project.setDisplayName(SugonSdnControllerConstant.ZSTACK_DEFAULT_ACCOUNT); project.setName(accountUuid); project.setUuid(accountUuid); - Status status = apiConnector.create(project); + Status status = client.create(project); if(status.isSuccess()){ logger.info("create tf project for zstack admin success"); completion.success(); @@ -81,7 +80,7 @@ public class SugonSdnController implements TfSdnController, SdnController { logger.warn("tf project for zstack admin already exists: " + accountUuid); completion.success(); } - } catch (IOException e) { + } catch (Exception e) { String message = String.format("create tf project for zstack admin on tf controller failed due to: %s", e.getMessage()); logger.error(message, e); completion.fail(operr(message)); @@ -135,8 +134,7 @@ public class SugonSdnController implements TfSdnController, SdnController { String name = l2NetworkVO.getName(); try { APICreateL2TfNetworkMsg l2TfNetworkMsg = (APICreateL2TfNetworkMsg) msg; - ApiConnector apiConnector = ApiConnectorFactory.build(sdnControllerVO.getIp(), SugonSdnControllerGlobalProperty.TF_CONTROLLER_PORT); - Project project = (Project) apiConnector.findById(Project.class, accountUuid); + Project project = (Project) client.findById(Project.class, accountUuid); if(project == null) { completion.fail(operr("get project[uuid:%s] on tf controller failed ", accountUuid)); }else{ @@ -149,7 +147,7 @@ public class SugonSdnController implements TfSdnController, SdnController { IPSegmentType ipSegmentType = new IPSegmentType(l2TfNetworkMsg.getIpPrefix(), l2TfNetworkMsg.getIpPrefixLength()); virtualNetwork.setIpSegment(ipSegmentType); } - Status status = apiConnector.create(virtualNetwork); + Status status = client.create(virtualNetwork); if(status.isSuccess()){ logger.info("create tf l2 network success, name:" + name); completion.success(); @@ -157,7 +155,7 @@ public class SugonSdnController implements TfSdnController, SdnController { completion.fail(operr("create tf l2 network[name:%s] on tf controller failed ", name)); } } - } catch (IOException e) { + } catch (Exception e) { String message = String.format("create tf l2 network[name:%s] on tf controller failed due to: %s", name, e.getMessage()); logger.error(message, e); completion.fail(operr(message)); @@ -169,13 +167,12 @@ public class SugonSdnController implements TfSdnController, SdnController { String uuid = StringDSL.transToTfUuid(l2NetworkVO.getUuid()); String name = l2NetworkVO.getName(); try { - ApiConnector apiConnector = ApiConnectorFactory.build(sdnControllerVO.getIp(), SugonSdnControllerGlobalProperty.TF_CONTROLLER_PORT); - VirtualNetwork virtualNetwork = (VirtualNetwork) apiConnector.findById(VirtualNetwork.class, uuid); + VirtualNetwork virtualNetwork = (VirtualNetwork) client.findById(VirtualNetwork.class, uuid); if(virtualNetwork == null){ completion.fail(operr("get virtual network[uuid:%s] on tf controller failed ", uuid)); }else{ virtualNetwork.setDisplayName(name); - Status status = apiConnector.update(virtualNetwork); + Status status = client.update(virtualNetwork); if(status.isSuccess()){ logger.info("update tf l2 network success, name:" + name); completion.success(); @@ -183,7 +180,7 @@ public class SugonSdnController implements TfSdnController, SdnController { completion.fail(operr("update tf l2 network[name:%s] on tf controller failed ", name)); } } - } catch (IOException e) { + } catch (Exception e) { String message = String.format("update tf l2 network[name:%s] on tf controller failed due to: %s ", name, e.getMessage()); logger.error(message, e); completion.fail(operr(message)); @@ -194,15 +191,14 @@ public class SugonSdnController implements TfSdnController, SdnController { public void deleteL2Network(L2NetworkVO l2NetworkVO, List systemTags, Completion completion) { String uuid = StringDSL.transToTfUuid(l2NetworkVO.getUuid()); try { - ApiConnector apiConnector = ApiConnectorFactory.build(sdnControllerVO.getIp(), SugonSdnControllerGlobalProperty.TF_CONTROLLER_PORT); - Status status = apiConnector.delete(VirtualNetwork.class, uuid); + Status status = client.delete(VirtualNetwork.class, uuid); if(status.isSuccess()){ logger.info("delete tf l2 network success, uuid:" + uuid); completion.success(); }else{ completion.fail(operr("delete tf l2 network[uuid:%s] on tf controller failed ", uuid)); } - } catch (IOException e) { + } catch (Exception e) { String message = String.format("delete tf l2 network[uuid:%s] on tf controller failed due to: %s", uuid, e.getMessage()); logger.error(message, e); completion.fail(operr(message)); @@ -296,9 +292,8 @@ public class SugonSdnController implements TfSdnController, SdnController { @Override public void deleteL3Network(L3NetworkVO l3NetworkVO, Completion completion) { try { - ApiConnector apiConnector = ApiConnectorFactory.build(sdnControllerVO.getIp(), SugonSdnControllerGlobalProperty.TF_CONTROLLER_PORT); // 获取 tf 网络信息 - VirtualNetwork vn = (VirtualNetwork)apiConnector.findById(VirtualNetwork.class, StringDSL.transToTfUuid(l3NetworkVO.getL2NetworkUuid())); + VirtualNetwork vn = (VirtualNetwork)client.findById(VirtualNetwork.class, StringDSL.transToTfUuid(l3NetworkVO.getL2NetworkUuid())); if(vn!=null){ // 判断tf网络是否存在 if(vn.getNetworkIpam()!=null) { @@ -313,7 +308,7 @@ public class SugonSdnController implements TfSdnController, SdnController { } } // 更新 tf 网络信息 - Status status = apiConnector.update(vn); + Status status = client.update(vn); if(!status.isSuccess()){ completion.fail(operr("delete tf l3 network[name:%s] on tf controller failed due to：%s", l3NetworkVO.getName(),status.getMsg())); // completion.fail(operr("delete tf l3 network[name:%s] on tf controller failed due to：%s", l3NetworkVO.getName(),"tf api call failed")); @@ -341,9 +336,8 @@ public class SugonSdnController implements TfSdnController, SdnController { @Override public void updateL3Network(L3NetworkVO l3NetworkVO, APIUpdateL3NetworkMsg msg, Completion completion) { try { - ApiConnector apiConnector = ApiConnectorFactory.build(sdnControllerVO.getIp(), SugonSdnControllerGlobalProperty.TF_CONTROLLER_PORT); // 获取 tf 网络信息 - VirtualNetwork vn = (VirtualNetwork)apiConnector.findById(VirtualNetwork.class, StringDSL.transToTfUuid(l3NetworkVO.getL2NetworkUuid())); + VirtualNetwork vn = (VirtualNetwork)client.findById(VirtualNetwork.class, StringDSL.transToTfUuid(l3NetworkVO.getL2NetworkUuid())); // 判断tf网络是否存在 if(vn!=null){ if(vn.getNetworkIpam()!=null) { @@ -362,10 +356,9 @@ public class SugonSdnController implements TfSdnController, SdnController { checkOp.get().addDnsNameservers(msg.getDnsDomain()); } // 更新 tf 网络信息 - Status status = apiConnector.update(vn); + Status status = client.update(vn); if(!status.isSuccess()){ completion.fail(operr("update tf l3 network[name:%s] on tf controller failed due to：%s", l3NetworkVO.getName(),status.getMsg())); -// completion.fail(operr("update tf l3 network[name:%s] on tf controller failed due to：%s", l3NetworkVO.getName(),"tf api call failed")); } else{ completion.success(); } @@ -392,9 +385,8 @@ public class SugonSdnController implements TfSdnController, SdnController { @Override public void addL3IpRangeByCidr(L3NetworkVO l3NetworkVO, APIAddIpRangeByNetworkCidrMsg msg, Completion completion) { try { - ApiConnector apiConnector = ApiConnectorFactory.build(sdnControllerVO.getIp(), SugonSdnControllerGlobalProperty.TF_CONTROLLER_PORT); // 获取 tf 网络信息 - VirtualNetwork vn = (VirtualNetwork) apiConnector.findById(VirtualNetwork.class, StringDSL.transToTfUuid(l3NetworkVO.getL2NetworkUuid())); + VirtualNetwork vn = (VirtualNetwork) client.findById(VirtualNetwork.class, StringDSL.transToTfUuid(l3NetworkVO.getL2NetworkUuid())); if(vn!=null){ IpamSubnetType ipamSubnetType = new IpamSubnetType(); // subnet_uuid @@ -438,9 +430,6 @@ public class SugonSdnController implements TfSdnController, SdnController { ipamSubnetType.addAllocationPools(allocationPoolType); // 设置分配IP从小到大 ipamSubnetType.setAddrFromStart(true); - // 封装实体 -> ObjectReference - IpamSubnets ipamSubnets = new IpamSubnets(); - ipamSubnets.addSubnets(ipamSubnetType); VnSubnetsType vnSubnetsType = new VnSubnetsType(); vnSubnetsType.addIpamSubnets(ipamSubnetType); if(vn.getNetworkIpam()!=null){ @@ -451,7 +440,7 @@ public class SugonSdnController implements TfSdnController, SdnController { vn.setNetworkIpam(networkIpam,vnSubnetsType); } // 更新 tf 网络信息 - Status status = apiConnector.update(vn); + Status status = client.update(vn); if(!status.isSuccess()){ completion.fail(operr("add tf l3 subnet[name:%s] on tf controller failed due to：%s", l3NetworkVO.getName(),status.getMsg())); // completion.fail(operr("add tf l3 subnet[name:%s] on tf controller failed due to：%s", l3NetworkVO.getName(),"tf api call failed")); @@ -473,9 +462,8 @@ public class SugonSdnController implements TfSdnController, SdnController { @Override public void addL3HostRoute(L3NetworkVO l3NetworkVO, APIAddHostRouteToL3NetworkMsg msg, Completion completion) { try { - ApiConnector apiConnector = ApiConnectorFactory.build(sdnControllerVO.getIp(), SugonSdnControllerGlobalProperty.TF_CONTROLLER_PORT); // 获取 tf 网络信息 - VirtualNetwork vn = (VirtualNetwork)apiConnector.findById(VirtualNetwork.class, StringDSL.transToTfUuid(l3NetworkVO.getL2NetworkUuid())); + VirtualNetwork vn = (VirtualNetwork)client.findById(VirtualNetwork.class, StringDSL.transToTfUuid(l3NetworkVO.getL2NetworkUuid())); if(vn!=null){ // 判断tf网络是否存在 if(vn.getNetworkIpam()!=null) { @@ -496,7 +484,7 @@ public class SugonSdnController implements TfSdnController, SdnController { // 替换host route checkOp.get().setHostRoutes(routeTableType); // 更新 tf 网络信息 - Status status = apiConnector.update(vn); + Status status = client.update(vn); if(!status.isSuccess()){ completion.fail(operr("add host router to l3 network[name:%s] on tf controller failed due to：%s", l3NetworkVO.getName(),status.getMsg())); // completion.fail(operr("add host router to l3 network[name:%s] on tf controller failed due to：%s", l3NetworkVO.getName(),"tf api call failed")); @@ -526,9 +514,8 @@ public class SugonSdnController implements TfSdnController, SdnController { @Override public void deleteL3HostRoute(L3NetworkVO l3NetworkVO, APIRemoveHostRouteFromL3NetworkMsg msg, Completion completion) { try { - ApiConnector apiConnector = ApiConnectorFactory.build(sdnControllerVO.getIp(), SugonSdnControllerGlobalProperty.TF_CONTROLLER_PORT); // 获取 tf 网络信息 - VirtualNetwork vn = (VirtualNetwork)apiConnector.findById(VirtualNetwork.class, StringDSL.transToTfUuid(l3NetworkVO.getL2NetworkUuid())); + VirtualNetwork vn = (VirtualNetwork)client.findById(VirtualNetwork.class, StringDSL.transToTfUuid(l3NetworkVO.getL2NetworkUuid())); if(vn!=null){ // 判断tf网络是否存在 if(vn.getNetworkIpam()!=null) { @@ -550,7 +537,7 @@ public class SugonSdnController implements TfSdnController, SdnController { // 替换host route checkOp.get().setHostRoutes(routeTableType); // 更新 tf 网络信息 - Status status = apiConnector.update(vn); + Status status = client.update(vn); if(!status.isSuccess()){ completion.fail(operr("delete host route from l3 network[name:%s] on tf controller failed due to：%s", l3NetworkVO.getName(),status.getMsg())); // completion.fail(operr("delete host route from l3 network[name:%s] on tf controller failed due to：%s", l3NetworkVO.getName(),"tf api call failed")); @@ -580,9 +567,8 @@ public class SugonSdnController implements TfSdnController, SdnController { @Override public void addL3Dns(L3NetworkVO l3NetworkVO, APIAddDnsToL3NetworkMsg msg, Completion completion) { try { - ApiConnector apiConnector = ApiConnectorFactory.build(sdnControllerVO.getIp(), SugonSdnControllerGlobalProperty.TF_CONTROLLER_PORT); // 获取 tf 网络信息 - VirtualNetwork vn = (VirtualNetwork)apiConnector.findById(VirtualNetwork.class, StringDSL.transToTfUuid(l3NetworkVO.getL2NetworkUuid())); + VirtualNetwork vn = (VirtualNetwork)client.findById(VirtualNetwork.class, StringDSL.transToTfUuid(l3NetworkVO.getL2NetworkUuid())); // 判断tf网络是否存在 if(vn!=null){ if(vn.getNetworkIpam()!=null) { @@ -608,7 +594,7 @@ public class SugonSdnController implements TfSdnController, SdnController { checkOp.get().setDhcpOptionList(dhcpOptionsListType); } // 更新 tf 网络信息 - Status status = apiConnector.update(vn); + Status status = client.update(vn); if(!status.isSuccess()){ completion.fail(operr("add dns to l3 network[name:%s] on tf controller failed due to：%s", l3NetworkVO.getName(),status.getMsg())); // completion.fail(operr("add dns to l3 network[name:%s] on tf controller failed due to：%s", l3NetworkVO.getName(),"tf api call failed")); @@ -638,9 +624,8 @@ public class SugonSdnController implements TfSdnController, SdnController { @Override public void deleteL3Dns(L3NetworkVO l3NetworkVO, APIRemoveDnsFromL3NetworkMsg msg, Completion completion) { try { - ApiConnector apiConnector = ApiConnectorFactory.build(sdnControllerVO.getIp(), SugonSdnControllerGlobalProperty.TF_CONTROLLER_PORT); // 获取 tf 网络信息 - VirtualNetwork vn = (VirtualNetwork)apiConnector.findById(VirtualNetwork.class, StringDSL.transToTfUuid(l3NetworkVO.getL2NetworkUuid())); + VirtualNetwork vn = (VirtualNetwork)client.findById(VirtualNetwork.class, StringDSL.transToTfUuid(l3NetworkVO.getL2NetworkUuid())); if(vn!=null){ // 判断tf网络是否存在 if(vn.getNetworkIpam()!=null) { @@ -654,7 +639,7 @@ public class SugonSdnController implements TfSdnController, SdnController { dnsValues.remove(msg.getDns()); checkOp.get().getDhcpOptionList().getDhcpOption().get(0).setDhcpOptionValue(StringUtils.join(dnsValues, " ")); // 更新 tf 网络信息 - Status status = apiConnector.update(vn); + Status status = client.update(vn); if (!status.isSuccess()) { completion.fail(operr("delete dns from to l3 network[name:%s] on tf controller failed due to：%s", l3NetworkVO.getName(),status.getMsg())); // completion.fail(operr("delete dns from to l3 network[name:%s] on tf controller failed due to：%s", l3NetworkVO.getName(),"tf api call failed")); diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/api/ApiConnectorImpl.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/api/ApiConnectorImpl.java index d8971f0dd0d1865335ed2d93f774467a11bfbdfb..ef88f3c9109fddf43d34425d3fb200b40ae3eadb 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/api/ApiConnectorImpl.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/api/ApiConnectorImpl.java @@ -225,6 +225,7 @@ class ApiConnectorImpl implements ApiConnector { request.setHeader("X-AUTH-TOKEN", _authtoken); } try { + _connection.setSocketTimeout(6000); _httpexecutor.preProcess(request, _httpproc, _httpcontext); response = _httpexecutor.execute(request, _connection, _httpcontext); response.setParams(_params); diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/api/TfCommands.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/api/TfCommands.java new file mode 100644 index 0000000000000000000000000000000000000000..849c26f405965249c1807668145c85a147f1b682 --- /dev/null +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/api/TfCommands.java @@ -0,0 +1,48 @@ +package org.zstack.sugonSdnController.controller.api; + +import org.zstack.utils.gson.JSONObjectUtil; + +import java.util.ArrayList; +import java.util.List; + +public class TfCommands { + public static final String TF_GET_DAEMON = "/fqname-to-id"; + public static final String TF_GET_DAEMON_DETAIL = "/domain/.*"; + public static final String TF_GET_PROJECT = "/project/.*"; + public static final String TF_CREATE_PROJECT = "/projects"; + public static final String TF_GET_NETWORK = "/virtual-network/.*"; + public static final String TF_CREATE_NETWORK = "/virtual-networks"; + public static final String TF_GET_VM = "/virtual-machine/.*"; + public static final String TF_CREATE_VM = "/virtual-machines"; + public static final String TF_GET_VMI = "/virtual-machine-interface/.*"; + public static final String TF_CREATE_VMI = "/virtual-machine-interfaces"; + public static final String TEST_DOMAIN_UUID = "cf8107ad-eee6-4a54-be5e-c05c96a9d552"; + public static final String TEST_PROJECT_UUID = "36c27e8f-f05c-4780-bf6d-2fa65700f22e"; + public static final String TEST_L2_UUID = "1eca756e-b935-45ed-a2bc-a3ebba535f7f"; + public static final String TEST_VM_UUID = "35c27fde-7f67-4c78-b80a-56ae1eefcb5b"; + public static final String TEST_VMI_UUID = "a581ee83-f5a5-4755-bedf-8d51f235da52"; + + public static class TfCmd { + public String toString() { + return JSONObjectUtil.toJsonString(this); + } + } + + public static class TfRsp { + } + + public static class GetDomainCmd extends TfCmd { + public String type; + public List fq_name = new ArrayList<>(); + } + + public static class GetDomainRsp extends TfRsp { + public String uuid; + } + + public static class GetProjectCmd extends TfCmd { + public String uuid; + } + +} + diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/api/TfHttpClient.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/api/TfHttpClient.java new file mode 100644 index 0000000000000000000000000000000000000000..00ae9db442968e81cc124e2f7352748976aa11fe --- /dev/null +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/api/TfHttpClient.java @@ -0,0 +1,403 @@ +package org.zstack.sugonSdnController.controller.api; + +import com.google.common.collect.ImmutableList; +import com.google.gson.*; +import org.apache.commons.lang.StringUtils; +import org.springframework.http.*; +import org.springframework.beans.factory.annotation.Autowire; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Configurable; +import org.springframework.web.util.UriComponentsBuilder; +import org.zstack.core.CoreGlobalProperty; +import org.zstack.core.MessageCommandRecorder; +import org.zstack.header.rest.RESTFacade; +import org.zstack.sugonSdnController.controller.SugonSdnControllerConstant; +import org.zstack.sugonSdnController.controller.SugonSdnControllerGlobalProperty; +import org.zstack.sugonSdnController.controller.api.types.Domain; +import org.zstack.utils.Utils; +import org.zstack.utils.gson.JSONObjectUtil; +import org.zstack.utils.logging.CLogger; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +@Configurable(preConstruction = true, autowire = Autowire.BY_TYPE) +public class TfHttpClient { + private static final CLogger logger = Utils.getLogger(TfHttpClient.class); + @Autowired + private RESTFacade restf; + private TimeUnit unit; + private Long timeout; + private String tf_ip; + private Map headers = new HashMap() { + { + put("Content-Type", "application/json; charset=UTF-8"); + } + }; + + public TfHttpClient(String ip) { + this.tf_ip = ip; + this.unit = TimeUnit.MILLISECONDS; + this.timeout = 30000l; + } + + public String getTypename(Class cls) { + String clsname = cls.getName(); + int loc = clsname.lastIndexOf('.'); + if (loc > 0) { + clsname = clsname.substring(loc + 1); + } + String typename = new String(); + for (int i = 0; i < clsname.length(); i++) { + char ch = clsname.charAt(i); + if (Character.isUpperCase(ch)) { + if (i > 0) { + typename += "-"; + } + ch = Character.toLowerCase(ch); + } + typename += ch; + } + return typename; + } + + public ApiObjectBase jsonToApiObject(String data, Class cls) { + if (data == null) { + return null; + } + final String typename = getTypename(cls); + final JsonParser parser = new JsonParser(); + final JsonObject js_obj = parser.parse(data).getAsJsonObject(); + if (js_obj == null) { + logger.warn("Unable to parse response"); + return null; + } + JsonElement element = null; + if (cls.getGenericSuperclass() == VRouterApiObjectBase.class) { + element = js_obj; + } else { + element = js_obj.get(typename); + } + if (element == null) { + logger.warn("Element " + typename + ": not found"); + return null; + } + ApiObjectBase resp = ApiSerializer.deserialize(element.toString(), cls); + return resp; + } + + public List jsonToApiObjects(String data, Class cls, boolean withDetail) { + if (data == null) { + return null; + } + final String typename = getTypename(cls); + List list = new ArrayList(); + final JsonParser parser = new JsonParser(); + final JsonObject js_obj= parser.parse(data).getAsJsonObject(); + if (js_obj == null) { + logger.warn("Unable to parse response"); + return null; + } + final JsonArray array = js_obj.getAsJsonArray(typename + "s"); + if (array == null) { + logger.warn("Element " + typename + ": not found"); + return null; + } + Gson json = ApiSerializer.getDeserializer(); + for (JsonElement element : array) { + ApiObjectBase obj; + if (withDetail) { + obj = jsonToApiObject(element.toString(), cls); + } else { + obj = json.fromJson(element.toString(), cls); + } + + if (obj == null) { + logger.warn("Unable to decode list element"); + continue; + } + list.add(obj); + } + return list; + } + + private String buildUrl(String ip, String path) { + UriComponentsBuilder ub = UriComponentsBuilder.newInstance(); + ub.scheme("http"); + if (CoreGlobalProperty.UNIT_TEST_ON) { + ub.host("localhost"); + ub.port(8989); + } else { + ub.host(ip); + ub.port(SugonSdnControllerGlobalProperty.TF_CONTROLLER_PORT); + } + + ub.path(path); + return ub.build().toUriString(); + } + + public String buildFqnJsonString(Class cls, List name_list) { + Gson json = new Gson(); + JsonObject js_dict = new JsonObject(); + js_dict.add("type", json.toJsonTree(getTypename(cls))); + js_dict.add("fq_name", json.toJsonTree(name_list)); + return js_dict.toString(); + } + + public String getUuid(String data) { + if (data == null) { + return null; + } + final JsonParser parser = new JsonParser(); + final JsonObject js_obj= parser.parse(data).getAsJsonObject(); + if (js_obj == null) { + logger.warn("Unable to parse response"); + return null; + } + final JsonElement element = js_obj.get("uuid"); + if (element == null) { + logger.warn("Element \"uuid\": not found"); + return null; + } + return element.getAsString(); + } + + private ResponseEntity execute(String url, HttpMethod method, String body) { + HttpHeaders requestHeaders = new HttpHeaders(); + if (headers != null) { + requestHeaders.setAll(headers); + } + requestHeaders.setContentType(MediaType.APPLICATION_JSON); + HttpEntity req = new HttpEntity(body, requestHeaders); + ResponseEntity response; + try { + response = restf.syncRawJson(buildUrl(tf_ip, url), req, method, unit, timeout); + } catch (Exception e){ + logger.warn(String.format("Execute http requests:%s failed, reason:%s", url, e.getMessage())); + return null; + } + + return response; + } + + public ApiObjectBase getDomain() { + TfCommands.GetDomainCmd getDomainCmd = new TfCommands.GetDomainCmd(); + getDomainCmd.type = getTypename(Domain.class); + List fqName = new ArrayList<>(); + fqName.add(SugonSdnControllerConstant.TF_DEFAULT_DOMAIN); + getDomainCmd.fq_name = fqName; + MessageCommandRecorder.record(getDomainCmd.getClass()); + String httpBody = JSONObjectUtil.toJsonString(getDomainCmd); + TfCommands.GetDomainRsp rsp = restf.syncJsonPost(buildUrl(tf_ip, TfCommands.TF_GET_DAEMON), httpBody, headers, TfCommands.GetDomainRsp.class, unit, timeout); + return findById(Domain.class, rsp.uuid); + } + + public synchronized ApiObjectBase findById(Class cls, String uuid) { + final String typename = getTypename(cls); + String url = String.format("/%s/%s", typename, uuid); + ResponseEntity response = execute(url, HttpMethod.GET, ""); + + if (response == null) { + return null; + } + + if (response.getStatusCode() != HttpStatus.OK) { + return null; + } + ApiObjectBase object = jsonToApiObject(response.getBody(), cls); + if (object == null) { + logger.warn("Unable to decode find response"); + } + + return object; + } + + public synchronized List listWithDetail(Class cls, + String fields, + String filters) { + final String typename = getTypename(cls); + String url = String.format("/%ss?detail=true", typename); + if (fields != null) { + url = url + "&fields=" + fields; + } + if (filters != null) { + url = url + "&filters=" + filters; + } + ResponseEntity response = execute(url, HttpMethod.GET, ""); + if (response == null) { + return null; + } + + if (response.getStatusCode() != HttpStatus.OK) { + logger.warn("list failed with :" + response.getBody()); + return null; + } + + String data = response.getBody(); + if (data == null) { + return null; + } + List list = jsonToApiObjects(data, cls, true); + if (list == null) { + logger.warn("Unable to parse/deserialize response: " + data); + } + return list; + } + + public List + getObjects(Class cls, List> refList) { + + List list = new ArrayList(); + for (ObjectReference ref : refList) { + ApiObjectBase obj = findById(cls, ref.getUuid()); + if (obj == null) { + logger.warn("Unable to find element with uuid: " + ref.getUuid()); + continue; + } + list.add(obj); + } + return list; + } + + public ApiObjectBase findByFQN(Class cls, String fullName) { + List fqn = ImmutableList.copyOf(StringUtils.split(fullName, ':')); + String uuid = findByName(cls, fqn); + if (uuid == null) { + return null; + } + return findById(cls, uuid); + } + + public synchronized String findByName(Class cls, List name_list) { + String jsonStr = buildFqnJsonString(cls, name_list); + ResponseEntity response = execute("/fqname-to-id", HttpMethod.POST, jsonStr); + + if (response == null) { + return null; + } + + if (response.getStatusCode() != HttpStatus.OK) { + return null; + } + + String data = response.getBody(); + if (data == null) { + return null; + } + logger.debug("<< Response Data: " + data); + + String uuid = getUuid(data); + if (uuid == null) { + logger.warn("Unable to parse response"); + return null; + } + return uuid; + } + + public synchronized Status create(ApiObjectBase obj) { + final String typename = getTypename(obj.getClass()); + final String jsdata = ApiSerializer.serializeObject(typename, obj); + String url; + + ResponseEntity response; + if (obj instanceof VRouterApiObjectBase) { + url = String.format("/%s", typename); + } else { + obj.updateQualifiedName(); + url = String.format("/%ss", typename); + } + response = execute(url, HttpMethod.POST, jsdata); + + if (response == null) { + return Status.failure("No response from API server."); + } + HttpStatus status = response.getStatusCode(); + if (status != HttpStatus.OK + && status != HttpStatus.CREATED + && status != HttpStatus.ACCEPTED ) { + + String reason = response.getBody(); + logger.error("create api request failed: " + reason); + if (status != HttpStatus.NOT_FOUND) { + logger.error("Failure message: " + reason); + } + return Status.failure(reason); + } + + ApiObjectBase resp = jsonToApiObject(response.getBody(), obj.getClass()); + if (resp == null) { + String reason = "Unable to decode Create response"; + logger.error(reason); + return Status.failure(reason); + } + + String uuid = obj.getUuid(); + if (uuid == null) { + obj.setUuid(resp.getUuid()); + } else if (!uuid.equals(resp.getUuid()) + && !(obj instanceof VRouterApiObjectBase)) { + logger.warn("Response contains unexpected uuid: " + resp.getUuid()); + return Status.success(); + } + logger.debug("Create " + typename + " uuid: " + obj.getUuid()); + return Status.success(); + } + + public synchronized Status update(ApiObjectBase obj) { + final String typename = getTypename(obj.getClass()); + final String jsdata = ApiSerializer.serializeObject(typename, obj); + String url = String.format("/%s/%s", typename, obj.getUuid()); + + ResponseEntity response = execute(url, HttpMethod.PUT, jsdata); + + if (response == null) { + return Status.failure("No response from API server."); + } + + HttpStatus status = response.getStatusCode(); + if (status != HttpStatus.OK + && status != HttpStatus.ACCEPTED ) { + String reason = response.getBody(); + logger.warn("<< Response:" + reason); + return Status.failure(reason); + } + + return Status.success(); + } + + public synchronized Status delete(Class cls, String uuid) { + if (findById(cls, uuid) == null) { + // object does not exist so we are ok + return Status.success(); + } + + final String typename = getTypename(cls); + String url = String.format("/%s/%s", typename, uuid); + ResponseEntity response = execute(url, HttpMethod.DELETE, ""); + + if (response == null) { + return Status.failure("No response from API server."); + } + + HttpStatus status = response.getStatusCode(); + if (status != HttpStatus.OK + && status != HttpStatus.NO_CONTENT + && status != HttpStatus.ACCEPTED ) { + String reason = response.getBody(); + logger.warn("Delete failed: " + reason); + if (status != HttpStatus.NOT_FOUND) { + logger.error("Failure message: " + response); + } + return Status.failure(reason); + } + return Status.success(); + } + + public Status delete(ApiObjectBase obj) throws IOException { + return delete(obj.getClass(), obj.getUuid()); + } +} diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/neutronClient/TfPortClient.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/neutronClient/TfPortClient.java index 4b923786a5fa6fe1abaa9ee9bd7681785631a620..dd10d8edec7bcac0dc04b51011ab505a5b098dae 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/neutronClient/TfPortClient.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/controller/neutronClient/TfPortClient.java @@ -22,7 +22,7 @@ import java.util.*; public class TfPortClient { private static final CLogger logger = Utils.getLogger(TfPortClient.class); - private ApiConnector apiConnector; + private TfHttpClient client; private String tenantId; @@ -33,10 +33,7 @@ public class TfPortClient { if (sdn == null){ throw new RuntimeException("Can not find a tf sdn controller."); } - apiConnector = ApiConnectorFactory.build(sdn.getIp(), SugonSdnControllerGlobalProperty.TF_CONTROLLER_PORT); - if (apiConnector == null) { - throw new RuntimeException(String.format("Can not connect to tf sdn controller: %s.", sdn.getIp())); - } + client = new TfHttpClient(sdn.getIp()); tenantId = StringDSL.transToTfUuid(sdn.getAccountUuid()); } @@ -56,14 +53,14 @@ public class TfPortClient { ipEntities.add(ipEntity); requestPortResourceEntity.setFixdIps(ipEntities); try { - VirtualNetwork netObj = (VirtualNetwork) apiConnector.findById(VirtualNetwork.class, l2Id); + VirtualNetwork netObj = (VirtualNetwork) client.findById(VirtualNetwork.class, l2Id); if (netObj == null) { throw new RuntimeException(String.format("Can not find tf virtualnetwork: %s.", l2Id)); } //if mac-address is specified, check against the exisitng ports //to see if there exists a port with the same mac-address if (!Objects.isNull(mac)) { - List ports = (List) apiConnector.listWithDetail( + List ports = (List) client.listWithDetail( VirtualMachineInterface.class, null, null); for (VirtualMachineInterface port : ports) { @@ -92,13 +89,13 @@ public class TfPortClient { port.setDisplayName(vmName); // always request for v4 and v6 ip object and handle the failure // create the object - Status result = apiConnector.create(port); + Status result = client.create(port); if (!result.isSuccess()) { throw new RuntimeException(String.format("Failed to create tf VirtualMachineInterface: %s, reason: %s", port.getUuid(), result.getMsg())); } // add support, nova boot --nic subnet-id=subnet_uuid - VirtualMachineInterface realPort = (VirtualMachineInterface) apiConnector.findById( + VirtualMachineInterface realPort = (VirtualMachineInterface) client.findById( VirtualMachineInterface.class, port.getUuid()); if (ip != null) { @@ -106,7 +103,7 @@ public class TfPortClient { portCreateInstanceIp(netObj, realPort, l3Id, ip); } catch (Exception e) { try { - apiConnector.delete(realPort); + client.delete(realPort); } catch (IOException ex) { throw new RuntimeException(ex); } @@ -125,7 +122,7 @@ public class TfPortClient { ipv4PortDelete = true; // failure in creating the instance ip. Roll back. try { - apiConnector.delete(realPort); + client.delete(realPort); } catch (IOException ex) { throw new RuntimeException(ex); } @@ -133,13 +130,13 @@ public class TfPortClient { } if (ipv4PortDelete) { try { - apiConnector.delete(realPort); + client.delete(realPort); } catch (IOException e) { throw new RuntimeException(e); } } } - VirtualMachineInterface newestPort = (VirtualMachineInterface) apiConnector.findById( + VirtualMachineInterface newestPort = (VirtualMachineInterface) client.findById( VirtualMachineInterface.class, realPort.getUuid()); return getPortResponse(newestPort); } catch (Exception e) { @@ -147,7 +144,7 @@ public class TfPortClient { } } - private TfPortResponse getPortResponse(VirtualMachineInterface portObj) throws IOException { + private TfPortResponse getPortResponse(VirtualMachineInterface portObj) { TfPortResponse tfPortResponse = new TfPortResponse(); tfPortResponse.setPortId(portObj.getUuid()); tfPortResponse.setCode(200); @@ -156,7 +153,7 @@ public class TfPortClient { List> ipBackRefs = portObj.getInstanceIpBackRefs(); if (ipBackRefs != null && ipBackRefs.size() > 0) { for (ObjectReference ipBackRef : ipBackRefs) { - InstanceIp ipObj = (InstanceIp) apiConnector.findById(InstanceIp.class, ipBackRef.getUuid()); + InstanceIp ipObj = (InstanceIp) client.findById(InstanceIp.class, ipBackRef.getUuid()); String ipAddr = ipObj != null ? ipObj.getAddress() : null; String subnetId = Objects.requireNonNull(ipObj).getSubnetUuid(); ipEntity.setIpAddress(ipAddr); @@ -169,7 +166,7 @@ public class TfPortClient { } private Status portCreateInstanceIp(VirtualNetwork virtualNetwork, VirtualMachineInterface port, - String subnetId, String ip) throws IOException { + String subnetId, String ip) { InstanceIp ipObj = new InstanceIp(); String ipFamily = "v4"; if (ip != null) { @@ -192,12 +189,12 @@ public class TfPortClient { PermType2 permType2 = new PermType2(); permType2.setOwner(tenantId); ipObj.setPerms2(permType2); - return apiConnector.create(ipObj); + return client.create(ipObj); } private VirtualMachineInterface getTfPortObject(TfPortRequestResource requestPortResourceEntity, VirtualNetwork virtualNetwork, String tfPortUUid) throws IOException { String projectId = requestPortResourceEntity.getTenantId(); - Project projectObj = (Project) apiConnector.findById(Project.class, projectId); + Project projectObj = (Project) client.findById(Project.class, projectId); IdPermsType idPermsType = new IdPermsType(); idPermsType.setEnable(true); String portUuid = String.valueOf(UUID.randomUUID()); @@ -231,11 +228,11 @@ public class TfPortClient { } public boolean ipInUseCheck(String ipAddr, String netId) throws IOException { - VirtualNetwork virtualNetwork = (VirtualNetwork) apiConnector.findById(VirtualNetwork.class, netId); + VirtualNetwork virtualNetwork = (VirtualNetwork) client.findById(VirtualNetwork.class, netId); if (virtualNetwork != null) { List> instanceIpBackRefs = virtualNetwork.getInstanceIpBackRefs(); if (instanceIpBackRefs != null) { - List ipObjects = (List) apiConnector.getObjects(InstanceIp.class, instanceIpBackRefs); + List ipObjects = (List) client.getObjects(InstanceIp.class, instanceIpBackRefs); if (ipObjects != null) { for (InstanceIp ipObj : ipObjects) { if (ipObj.getAddress().equals(ipAddr)) { @@ -252,7 +249,7 @@ public class TfPortClient { private VirtualMachine ensureInstanceExists(String deviceId, String projectId, boolean baremeetal) throws IOException { VirtualMachine instanceObj = null; try { - instanceObj = (VirtualMachine) apiConnector.findById(VirtualMachine.class, deviceId); + instanceObj = (VirtualMachine) client.findById(VirtualMachine.class, deviceId); if (instanceObj != null) { return instanceObj; } @@ -267,18 +264,18 @@ public class TfPortClient { } else { instanceObj.setServerType("virtual-server"); } - apiConnector.create(instanceObj); + client.create(instanceObj); } catch (Exception e) { // Exception...... VirtualMachine dbInstanceObj = null; if (instanceObj.getUuid() != null) { - dbInstanceObj = (VirtualMachine) apiConnector.findById(VirtualMachine.class, instanceObj.getUuid()); + dbInstanceObj = (VirtualMachine) client.findById(VirtualMachine.class, instanceObj.getUuid()); } else { - dbInstanceObj = (VirtualMachine) apiConnector.findByFQN(VirtualMachine.class, instanceObj.getName()); + dbInstanceObj = (VirtualMachine) client.findByFQN(VirtualMachine.class, instanceObj.getName()); } if (dbInstanceObj != null) { if (baremeetal && !Objects.equals(dbInstanceObj.getServerType(), "baremetal-server")) { - apiConnector.update(instanceObj); + client.update(instanceObj); } else { instanceObj = dbInstanceObj; } @@ -289,24 +286,24 @@ public class TfPortClient { public TfPortResponse getVirtualMachineInterface(String portId) { try { - VirtualMachineInterface port = (VirtualMachineInterface) apiConnector.findById( + VirtualMachineInterface port = (VirtualMachineInterface) client.findById( VirtualMachineInterface.class, portId); if (port != null){ return getPortResponse(port); }else { return null; } - } catch (IOException e) { + } catch (Exception e) { throw new RuntimeException(e); } } public List getVirtualMachineInterfaceDetail() { try { - List ports = (List) apiConnector.listWithDetail( + List ports = (List) client.listWithDetail( VirtualMachineInterface.class, null, null); return ports; - } catch (IOException e) { + } catch (Exception e) { throw new RuntimeException(e); } } @@ -314,7 +311,7 @@ public class TfPortClient { public TfPortResponse deletePort(String portId) { TfPortResponse response = new TfPortResponse(); try { - VirtualMachineInterface portObj = (VirtualMachineInterface) apiConnector.findById(VirtualMachineInterface.class, portId); + VirtualMachineInterface portObj = (VirtualMachineInterface) client.findById(VirtualMachineInterface.class, portId); if (portObj == null) { response.setCode(200); return response; @@ -323,18 +320,18 @@ public class TfPortClient { List> iipBackRefs = portObj.getInstanceIpBackRefs(); if (iipBackRefs != null && iipBackRefs.size() > 0) { for (ObjectReference iipBackRef : iipBackRefs) { - InstanceIp iipObj = (InstanceIp) apiConnector.findById(InstanceIp.class, iipBackRef.getUuid()); + InstanceIp iipObj = (InstanceIp) client.findById(InstanceIp.class, iipBackRef.getUuid()); // in case of shared ip only delete the link to the VMI if (iipObj != null) { iipObj.removeVirtualMachineInterface(portObj); List> virtualMachineInterface = iipObj.getVirtualMachineInterface(); if (virtualMachineInterface == null || virtualMachineInterface.size() == 0) { - Status delResult = apiConnector.delete(InstanceIp.class, iipBackRef.getUuid()); + Status delResult = client.delete(InstanceIp.class, iipBackRef.getUuid()); if (!delResult.isSuccess()) { throw new RuntimeException("Tf instance ip delete failed: " + iipBackRef.getUuid()); } } else { - apiConnector.update(iipObj); + client.update(iipObj); } } } @@ -345,12 +342,12 @@ public class TfPortClient { for (ObjectReference fipBackRef : fipBackRefs) { FloatingIp fipObj = getTfFloatingipObject(fipBackRef.getUuid()); if (fipObj != null) { - apiConnector.update(fipObj); + client.update(fipObj); } } } - apiConnector.delete(VirtualMachineInterface.class, portId); + client.delete(VirtualMachineInterface.class, portId); // delete VirtualMachine if this was the last port String instanceId; if (Objects.equals(portObj.getParentType(), "virtual-machine")) { @@ -364,9 +361,9 @@ public class TfPortClient { } } if (instanceId != null) { - VirtualMachine vm = (VirtualMachine) apiConnector.findById(VirtualMachine.class, instanceId); + VirtualMachine vm = (VirtualMachine) client.findById(VirtualMachine.class, instanceId); if (CollectionUtils.isEmpty(vm.getVirtualMachineInterfaceBackRefs())) { - apiConnector.delete(VirtualMachine.class, instanceId); + client.delete(VirtualMachine.class, instanceId); } } response.setCode(200); @@ -380,8 +377,8 @@ public class TfPortClient { } } - private FloatingIp getTfFloatingipObject(String uuid) throws IOException { - FloatingIp fipObj = (FloatingIp) apiConnector.findById(FloatingIp.class, uuid); + private FloatingIp getTfFloatingipObject(String uuid) { + FloatingIp fipObj = (FloatingIp) client.findById(FloatingIp.class, uuid); if (fipObj == null) { return null; } @@ -399,7 +396,7 @@ public class TfPortClient { */ public boolean checkTfIpAvailability(String ipAddr, String subnetId) throws IOException { L3NetworkVO l3Network = Q.New(L3NetworkVO.class).eq(L3NetworkVO_.uuid, subnetId).find(); - VirtualNetwork virtualNetwork = (VirtualNetwork) apiConnector.findById( + VirtualNetwork virtualNetwork = (VirtualNetwork) client.findById( VirtualNetwork.class, StringDSL.transToTfUuid(l3Network.getL2NetworkUuid())); if (Objects.isNull(virtualNetwork)) { @@ -413,11 +410,11 @@ public class TfPortClient { List> floatingIpPools = virtualNetwork.getFloatingIpPools(); if (CollectionUtils.isNotEmpty(floatingIpPools)) { for (ObjectReference floatingIpPool : floatingIpPools) { - FloatingIpPool floatingIpPoolObj = (FloatingIpPool) apiConnector.findById(FloatingIpPool.class, floatingIpPool.getUuid()); + FloatingIpPool floatingIpPoolObj = (FloatingIpPool) client.findById(FloatingIpPool.class, floatingIpPool.getUuid()); List> floatingIps = floatingIpPoolObj.getFloatingIps(); if (CollectionUtils.isNotEmpty(floatingIps)) { for (ObjectReference fip : floatingIps) { - FloatingIp fipObj = (FloatingIp) apiConnector.findById(FloatingIp.class, fip.getUuid()); + FloatingIp fipObj = (FloatingIp) client.findById(FloatingIp.class, fip.getUuid()); if (fipObj.getAddress().equals(ipAddr)) { return true; } @@ -427,7 +424,7 @@ public class TfPortClient { } } else { // else instance ips. if (CollectionUtils.isNotEmpty(instanceIpBackRefs)) { - List ipObjects = (List) apiConnector.getObjects(InstanceIp.class, instanceIpBackRefs); + List ipObjects = (List) client.getObjects(InstanceIp.class, instanceIpBackRefs); // check all instance ips. if (CollectionUtils.isNotEmpty(ipObjects)) { for (InstanceIp ipObj : ipObjects) { @@ -480,11 +477,11 @@ public class TfPortClient { public void updateTfPort(String tfPortUUid, String accountId, String deviceId) { try { - VirtualMachineInterface port = (VirtualMachineInterface) apiConnector.findById( + VirtualMachineInterface port = (VirtualMachineInterface) client.findById( VirtualMachineInterface.class, tfPortUUid); VirtualMachine vm = ensureInstanceExists(deviceId, accountId, false); port.setVirtualMachine(vm); - apiConnector.update(port); + client.update(port); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/header/SugonApiInterceptor.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/header/SugonApiInterceptor.java new file mode 100644 index 0000000000000000000000000000000000000000..18294d7463a8705e517bec26da11355a10341dfb --- /dev/null +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/header/SugonApiInterceptor.java @@ -0,0 +1,89 @@ +package org.zstack.sugonSdnController.header; + +import org.springframework.beans.factory.annotation.Autowired; +import org.zstack.core.cloudbus.CloudBus; +import org.zstack.core.db.Q; +import org.zstack.header.apimediator.ApiMessageInterceptionException; +import org.zstack.header.apimediator.ApiMessageInterceptor; +import org.zstack.header.apimediator.GlobalApiMessageInterceptor; +import org.zstack.header.apimediator.StopRoutingException; +import org.zstack.header.message.APIMessage; +import org.zstack.header.network.l2.*; +import org.zstack.header.network.l3.*; +import org.zstack.header.vm.VmInstanceConstant; +import org.zstack.header.vm.VmNicVO; +import org.zstack.header.vm.VmNicVO_; +import org.zstack.sdnController.header.APIRemoveSdnControllerEvent; +import org.zstack.sdnController.header.APIRemoveSdnControllerMsg; +import org.zstack.sugonSdnController.controller.SugonSdnControllerConstant; + +import java.util.ArrayList; +import java.util.List; + +import static org.zstack.core.Platform.operr; + + +public class SugonApiInterceptor implements ApiMessageInterceptor, GlobalApiMessageInterceptor { + @Autowired + private CloudBus bus; + + @Override + public APIMessage intercept(APIMessage msg) throws ApiMessageInterceptionException { + if (msg instanceof APIDeleteL2NetworkMsg) { + validate((APIDeleteL2NetworkMsg) msg); + } else if (msg instanceof APIDeleteL3NetworkMsg) { + validate((APIDeleteL3NetworkMsg) msg); + } else if (msg instanceof APIRemoveSdnControllerMsg) { + validate((APIRemoveSdnControllerMsg) msg); + } + + return msg; + } + + private void validate(APIDeleteL2NetworkMsg msg) { + APIDeleteL2NetworkEvent evt = new APIDeleteL2NetworkEvent(msg.getId()); + if(Q.New(L3NetworkVO.class).eq(L3NetworkVO_.l2NetworkUuid, msg.getL2NetworkUuid()) + .eq(L3NetworkVO_.type, SugonSdnControllerConstant.L3_TF_NETWORK_TYPE).count() > 0){ + String error = String.format("L2Network[%s] still has some L3Networks, please delete L3Networks first.", + msg.getL2NetworkUuid()); + evt.setError(operr(error)); + bus.publish(evt); + throw new StopRoutingException(); + } + } + + private void validate(APIDeleteL3NetworkMsg msg) { + APIDeleteL3NetworkEvent evt = new APIDeleteL3NetworkEvent(msg.getId()); + if(Q.New(VmNicVO.class).eq(VmNicVO_.l3NetworkUuid, msg.getL3NetworkUuid()) + .eq(VmNicVO_.type, VmInstanceConstant.TF_VIRTUAL_NIC_TYPE).count() > 0){ + String error = String.format("L3Network[%s] still has some Nics, please delete all Nics first.", + msg.getId()); + evt.setError(operr(error)); + bus.publish(evt); + throw new StopRoutingException(); + } + } + + private void validate(APIRemoveSdnControllerMsg msg) { + APIRemoveSdnControllerEvent evt = new APIRemoveSdnControllerEvent(msg.getId()); + if(Q.New(L2NetworkVO.class).eq(L2NetworkVO_.type, SugonSdnControllerConstant.L2_TF_NETWORK_TYPE).count() > 0){ + String error = String.format("There are some TfL2Networks exists, please delete all TfL2Networks first.", + msg.getId()); + evt.setError(operr(error)); + bus.publish(evt); + throw new StopRoutingException(); + } + } + + public List getMessageClassToIntercept() { + List ret = new ArrayList<>(); + ret.add(APIDeleteL2NetworkMsg.class); + ret.add(APIDeleteL3NetworkMsg.class); + ret.add(APIRemoveSdnControllerMsg.class); + return ret; + } + + public InterceptorPosition getPosition() { + return InterceptorPosition.END; + } +} diff --git a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfL2Network.java b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfL2Network.java index c6ee8351ed7d59f49e2ee99b938a1af2008b26a0..06520d030a738b1871bde710e7c91f1af1e383aa 100644 --- a/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfL2Network.java +++ b/plugin/sugonSdnController/src/main/java/org/zstack/sugonSdnController/network/TfL2Network.java @@ -193,13 +193,6 @@ public class TfL2Network extends L2NoVlanNetwork implements TfL2NetworkExtension private void handle(APIDeleteL2NetworkMsg msg) { APIDeleteL2NetworkEvent evt = new APIDeleteL2NetworkEvent(msg.getId()); - if(Q.New(L3NetworkVO.class).eq(L3NetworkVO_.l2NetworkUuid, msg.getL2NetworkUuid()).count() > 0){ - String error = String.format("L2Network[%s] still has some L3Networks, please delete L3Networks first.", - msg.getL2NetworkUuid()); - evt.setError(operr(error)); - bus.publish(evt); - return; - } deleteTfL2NetworkOnSdnController(self, new Completion(msg) { @Override public void success() { diff --git a/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SdnControllerTest.groovy b/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SdnControllerTest.groovy index 9895cb45b1c888a3eac90bd1a7f3b0e1ec26d922..6d2c5bbf310c70faea5b49b1fbd98bebd85bbd54 100644 --- a/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SdnControllerTest.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SdnControllerTest.groovy @@ -19,6 +19,8 @@ class SdnControllerTest extends Test { include("vip.xml") include("vxlan.xml") include("sdnController.xml") + include("sugonSdnController.xml") + include("TfPortAllocator.xml") eip() lb() portForwarding() diff --git a/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy b/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy new file mode 100644 index 0000000000000000000000000000000000000000..141813024d13f1f2dae8a7cbc5a62cf8753016b8 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerCase.groovy @@ -0,0 +1,232 @@ +package org.zstack.test.integration.network.sdnController + +import org.zstack.core.db.DatabaseFacade +import org.zstack.sdk.* +import org.zstack.sugonSdnController.controller.SugonSdnControllerConstant +import org.zstack.sugonSdnController.controller.api.types.MacAddressesType +import org.zstack.sugonSdnController.controller.api.types.Project +import org.zstack.sugonSdnController.controller.api.types.VirtualMachineInterface +import org.zstack.sugonSdnController.controller.api.ApiSerializer +import org.zstack.sugonSdnController.controller.api.TfCommands +import org.zstack.sdnController.header.SdnControllerVO +import org.zstack.header.network.l3.L3NetworkVO; +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.SubCase +import javax.persistence.TypedQuery; +import org.springframework.http.ResponseEntity +import org.springframework.http.HttpStatus + + +class SugonSdnControllerCase extends SubCase { + EnvSpec env + DatabaseFacade dbf + + @Override + void setup() { + spring { + useSpring(SdnControllerTest.springSpec) + } + } + + @Override + void environment() { + env = SugonSdnControllerEnv.SdnControllerBasicEnv() + } + + @Override + void test() { + env.create { + dbf = bean(DatabaseFacade.class) + testTfApi() + } + } + + @Override + void clean() { + env.delete() + } + + void testTfApi() { + def zone = env.inventoryByName("zone") as ZoneInventory + def cluster1 = env.inventoryByName("cluster1") as ClusterInventory + def cluster2 = env.inventoryByName("cluster2") as ClusterInventory + def instanceOffering = env.inventoryByName("instanceOffering") as InstanceOfferingInventory + def image = env.inventoryByName("image1") as ImageInventory + + String sql = "select sdn" + + " from SdnControllerVO sdn" + + " where sdn.vendorType = :vendorType"; + TypedQuery q = dbf.getEntityManager().createQuery(sql, SdnControllerVO.class); + q.setParameter("vendorType", SugonSdnControllerConstant.TF_CONTROLLER); + List sdns = q.getResultList(); + SdnControllerVO sdn = sdns.get(0); + updateSdnController { + uuid = sdn.uuid + name = "sugon_sdn" + description = "sugon sdn:tf" + } + SdnControllerVO vo = dbf.findByUuid(sdn.uuid, SdnControllerVO.class) + assert vo.name == "sugon_sdn" + assert vo.description == "sugon sdn:tf" + + L2NetworkInventory l2TfNetwork = createL2TfNetwork { + name = "tfL2Network" + type = SugonSdnControllerConstant.L2_TF_NETWORK_TYPE + physicalInterface = sdn.uuid + zoneUuid = zone.uuid + } + + updateL2Network { + uuid = l2TfNetwork.uuid + name = "test_tf_l2" + description = "test tf l2 network" + } + + attachL2NetworkToCluster { + l2NetworkUuid = l2TfNetwork.uuid + clusterUuid = cluster1.uuid + } + + attachL2NetworkToCluster { + l2NetworkUuid = l2TfNetwork.uuid + clusterUuid = cluster2.uuid + } + + detachL2NetworkFromCluster { + l2NetworkUuid = l2TfNetwork.uuid + clusterUuid = cluster1.uuid + } + + List l2TfNetworks = queryL2Network { + conditions=["type=" + SugonSdnControllerConstant.L2_TF_NETWORK_TYPE]} + assert l2TfNetworks.size() == 1 + + L3NetworkInventory test_l3_1 = createL3Network { + l2NetworkUuid = l2TfNetwork.uuid + name = "test-l3" + type = SugonSdnControllerConstant.L3_TF_NETWORK_TYPE + } + + List l3TfNetworks = queryL3Network { + conditions=["type=" + SugonSdnControllerConstant.L3_TF_NETWORK_TYPE]} + assert l3TfNetworks.size() == 1 + + updateL3Network { + uuid = test_l3_1.uuid + name = "test_l3_1" + description = "test_l3_1" + } + L3NetworkVO l3Network = dbf.findByUuid(test_l3_1.uuid, L3NetworkVO.class) + assert l3Network.name == "test_l3_1" + assert l3Network.description == "test_l3_1" + + addIpRangeByNetworkCidr { + name = "Test-IPRange" + networkCidr = "192.168.10.0/24" + l3NetworkUuid = test_l3_1.uuid + } + + L3NetworkInventory l3Inv = addDnsToL3Network { + l3NetworkUuid = test_l3_1.uuid + dns = "1.1.1.1" + } + assert l3Inv.dns.get(0) == '1.1.1.1' + + l3Inv = removeDnsFromL3Network { + l3NetworkUuid = test_l3_1.uuid + dns = "1.1.1.1" + } + assert l3Inv.dns == null + + VmInstanceInventory vm = createVmInstance { + name = "test-vm1" + instanceOfferingUuid = instanceOffering.uuid + imageUuid = image.uuid + l3NetworkUuids = [test_l3_1.uuid] + } + + L3NetworkInventory test_l3_2 = createL3Network { + l2NetworkUuid = l2TfNetwork.uuid + name = "test-l3-2" + type = SugonSdnControllerConstant.L3_TF_NETWORK_TYPE + } + + addIpRangeByNetworkCidr { + name = "Test-IPRange2" + networkCidr = "192.168.20.0/24" + l3NetworkUuid = test_l3_2.uuid + } + + env.simulator(TfCommands.TF_GET_VMI) { + VirtualMachineInterface rsp = new VirtualMachineInterface(); + rsp.name = "5255167d-46c8-4d9e-b4b1-e20c38ce25d9" + rsp.uuid = "5255167d-46c8-4d9e-b4b1-e20c38ce25d9" + List macList = new ArrayList(); + macList.add("08:00:27:b4:e1:98"); + MacAddressesType macAddress = new MacAddressesType(macList); + rsp.setMacAddresses(macAddress); + Project project = new Project(); + project.name = TfCommands.TEST_PROJECT_UUID + project.uuid = TfCommands.TEST_PROJECT_UUID + project.displayName = "admin"; + rsp.parent = project + rsp.instance_ip_back_refs = null + String json = ApiSerializer.serializeObject("virtual-machine-interface", rsp); + ResponseEntity response = new ResponseEntity(json, HttpStatus.OK); + return response.getBody() + } + + attachL3NetworkToVm { + l3NetworkUuid = test_l3_2.uuid + vmInstanceUuid = vm.uuid + } + VmInstanceInventory result = queryVmInstance { + conditions = ["uuid=${vm.uuid}"] + }[0] + assert result.vmNics.size() == 2 + + detachL3NetworkFromVm { + vmNicUuid = vm.getVmNics().get(0).uuid + } + result = queryVmInstance { + conditions = ["uuid=${vm.uuid}"] + }[0] + assert result.vmNics.size() == 1 + + destroyVmInstance { + uuid = vm.uuid + } + + expungeVmInstance{ + uuid = vm.uuid + } + + deleteL3Network { + delegate.uuid = test_l3_1.uuid + } + + deleteL3Network { + delegate.uuid = test_l3_2.uuid + } + + l3TfNetworks = queryL3Network {conditions=["type=" + SugonSdnControllerConstant.L3_TF_NETWORK_TYPE]} + assert l3TfNetworks.size() == 0 + + expectError { + removeSdnController { + uuid = sdn.uuid + } + } + + deleteL2Network { + delegate.uuid = l2TfNetwork.uuid + } + + l2TfNetworks = queryL2Network {conditions=["type=" + SugonSdnControllerConstant.L2_TF_NETWORK_TYPE]} + assert l2TfNetworks.size() == 0 + + removeSdnController { + uuid = sdn.uuid + } + } +} diff --git a/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerEnv.groovy b/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerEnv.groovy new file mode 100644 index 0000000000000000000000000000000000000000..00623ff866b77f51472218ae20e6d71a57dc86b0 --- /dev/null +++ b/test/src/test/groovy/org/zstack/test/integration/network/sdnController/SugonSdnControllerEnv.groovy @@ -0,0 +1,94 @@ +package org.zstack.test.integration.network.sdnController + +import org.zstack.testlib.EnvSpec +import org.zstack.testlib.Test +import org.zstack.utils.data.SizeUnit + +class SugonSdnControllerEnv { + static EnvSpec SdnControllerBasicEnv() { + return Test.makeEnv { + instanceOffering { + name = "instanceOffering" + memory = SizeUnit.GIGABYTE.toByte(1) + cpu = 1 + } + + sftpBackupStorage { + name = "sftp" + url = "/sftp" + username = "root" + password = "password" + hostname = "localhost" + + image { + name = "image1" + url = "http://zstack.org/download/test.qcow2" + } + + image { + name = "vr" + url = "http://zstack.org/download/vr.qcow2" + } + } + + zone { + name = "zone" + description = "test" + + cluster { + name = "cluster1" + hypervisorType = "KVM" + + kvm { + name = "kvm1" + managementIp = "127.0.0.1" + username = "root" + password = "password" + } + + kvm { + name = "kvm2" + managementIp = "127.0.0.2" + username = "root" + password = "password" + } + + kvm { + name = "kvm3" + managementIp = "127.0.0.3" + username = "root" + password = "password" + } + attachPrimaryStorage("local") + } + + cluster { + name = "cluster2" + hypervisorType = "KVM" + + kvm { + name = "kvm4" + managementIp = "127.0.0.4" + username = "root" + password = "password" + } + attachPrimaryStorage("local") + } + attachBackupStorage("sftp") + + localPrimaryStorage { + name = "local" + url = "/local_ps" + } + + sdnController { + vendorType = "TF" + name = "tf" + ip = "127.0.0.1" + userName = "user" + password = "password" + } + } + } + } +} diff --git a/testlib/src/main/java/org/zstack/testlib/SdnControllerSpec.groovy b/testlib/src/main/java/org/zstack/testlib/SdnControllerSpec.groovy index a00b3d455888a54151a1c20492d4848fc8d631ed..a8e3d148c64bf732a0368642e4c5a9c5f40d30bc 100644 --- a/testlib/src/main/java/org/zstack/testlib/SdnControllerSpec.groovy +++ b/testlib/src/main/java/org/zstack/testlib/SdnControllerSpec.groovy @@ -1,6 +1,7 @@ package org.zstack.testlib - +import org.springframework.http.HttpStatus +import org.springframework.http.ResponseEntity import org.zstack.sdk.SdnControllerInventory import org.zstack.sdnController.h3cVcfc.H3cVcfcCommands import org.zstack.sdnController.h3cVcfc.H3cVcfcCommands.LoginReply @@ -13,7 +14,15 @@ import org.zstack.sdnController.h3cVcfc.H3cVcfcCommands.VniRangeStruct import org.zstack.sdnController.h3cVcfc.H3cVcfcCommands.GetH3cTenantsRsp import org.zstack.sdnController.h3cVcfc.H3cVcfcCommands.H3cTenantStruct import org.zstack.sdnController.h3cVcfc.H3cVcfcCommands.GetH3cTeamLederIpReply -import org.zstack.testlib.* +import org.zstack.sugonSdnController.controller.SugonSdnControllerConstant +import org.zstack.sugonSdnController.controller.api.ApiSerializer +import org.zstack.sugonSdnController.controller.api.TfCommands +import org.zstack.sugonSdnController.controller.api.types.Domain +import org.zstack.sugonSdnController.controller.api.types.MacAddressesType +import org.zstack.sugonSdnController.controller.api.types.Project +import org.zstack.sugonSdnController.controller.api.types.VirtualMachine +import org.zstack.sugonSdnController.controller.api.types.VirtualMachineInterface +import org.zstack.sugonSdnController.controller.api.types.VirtualNetwork /** * Created by shixin.ruan on 2019/09/26. @@ -127,6 +136,111 @@ class SdnControllerSpec extends Spec implements HasSession { rsp.ip = "127.1.1.1" return rsp } + + simulator(TfCommands.TF_GET_DAEMON) { + TfCommands.GetDomainRsp rsp = new TfCommands.GetDomainRsp() + rsp.uuid = TfCommands.TEST_DOMAIN_UUID + return rsp + } + + simulator(TfCommands.TF_GET_DAEMON_DETAIL) { + Domain rsp = new Domain() + rsp.uuid = TfCommands.TEST_DOMAIN_UUID + rsp.name = SugonSdnControllerConstant.TF_DEFAULT_DOMAIN + String json = ApiSerializer.serializeObject("domain", rsp); + ResponseEntity response = new ResponseEntity(json, HttpStatus.OK); + return response.getBody() + } + + simulator(TfCommands.TF_GET_PROJECT) { + Project rsp = new Project(); + rsp.name = TfCommands.TEST_PROJECT_UUID + rsp.uuid = TfCommands.TEST_PROJECT_UUID + rsp.displayName = "admin"; + String json = ApiSerializer.serializeObject("project", rsp); + ResponseEntity response = new ResponseEntity(json, HttpStatus.OK); + + return response.getBody() + } + + simulator(TfCommands.TF_CREATE_PROJECT) { + Project rsp = new Project(); + rsp.name = TfCommands.TEST_PROJECT_UUID + rsp.uuid = TfCommands.TEST_PROJECT_UUID + rsp.displayName = "admin"; + String json = ApiSerializer.serializeObject("project", rsp); + ResponseEntity response = new ResponseEntity(json, HttpStatus.OK); + return response.getBody() + } + + simulator(TfCommands.TF_CREATE_NETWORK) { + VirtualNetwork rsp = new VirtualNetwork(); + rsp.name = TfCommands.TEST_L2_UUID + rsp.uuid = TfCommands.TEST_L2_UUID + String json = ApiSerializer.serializeObject("virtual-network", rsp); + ResponseEntity response = new ResponseEntity(json, HttpStatus.OK); + return response.getBody() + } + + simulator(TfCommands.TF_GET_NETWORK) { + VirtualNetwork rsp = new VirtualNetwork(); + rsp.name = TfCommands.TEST_L2_UUID + rsp.uuid = TfCommands.TEST_L2_UUID + String json = ApiSerializer.serializeObject("virtual-network", rsp); + ResponseEntity response = new ResponseEntity(json, HttpStatus.OK); + return response.getBody() + } + + simulator(TfCommands.TF_CREATE_VM) { + VirtualMachine rsp = new VirtualMachine(); + rsp.name = TfCommands.TEST_VM_UUID + rsp.uuid = TfCommands.TEST_VM_UUID + String json = ApiSerializer.serializeObject("virtual-machine", rsp); + ResponseEntity response = new ResponseEntity(json, HttpStatus.OK); + return response.getBody() + } + + simulator(TfCommands.TF_GET_VM) { + VirtualMachine rsp = new VirtualMachine(); + rsp.name = TfCommands.TEST_VM_UUID + rsp.uuid = TfCommands.TEST_VM_UUID + String json = ApiSerializer.serializeObject("virtual-machine", rsp); + ResponseEntity response = new ResponseEntity(json, HttpStatus.OK); + return response.getBody() + } + + simulator(TfCommands.TF_CREATE_VMI) { + VirtualMachineInterface rsp = new VirtualMachineInterface(); + rsp.name = TfCommands.TEST_VMI_UUID + rsp.uuid = TfCommands.TEST_VMI_UUID + Project project = new Project(); + project.name = TfCommands.TEST_PROJECT_UUID + project.uuid = TfCommands.TEST_PROJECT_UUID + project.displayName = "admin"; + rsp.setParent(project) + String json = ApiSerializer.serializeObject("virtual-machine-interface", rsp); + ResponseEntity response = new ResponseEntity(json, HttpStatus.OK); + return response.getBody() + } + + simulator(TfCommands.TF_GET_VMI) { + VirtualMachineInterface rsp = new VirtualMachineInterface(); + rsp.name = TfCommands.TEST_VMI_UUID + rsp.uuid = TfCommands.TEST_VMI_UUID + List macList = new ArrayList(); + macList.add("08:00:27:b4:e1:99"); + MacAddressesType macAddress = new MacAddressesType(macList); + rsp.setMacAddresses(macAddress); + Project project = new Project(); + project.name = TfCommands.TEST_PROJECT_UUID + project.uuid = TfCommands.TEST_PROJECT_UUID + project.displayName = "admin"; + rsp.parent = project + rsp.instance_ip_back_refs = null + String json = ApiSerializer.serializeObject("virtual-machine-interface", rsp); + ResponseEntity response = new ResponseEntity(json, HttpStatus.OK); + return response.getBody() + } } } diff --git a/testlib/src/main/java/org/zstack/testlib/Test.groovy b/testlib/src/main/java/org/zstack/testlib/Test.groovy index 96194c356165bc9a0a885bcbe84296b4e275361c..47803b40ca008617ac429873fc15e2252841766e 100755 --- a/testlib/src/main/java/org/zstack/testlib/Test.groovy +++ b/testlib/src/main/java/org/zstack/testlib/Test.groovy @@ -149,6 +149,8 @@ abstract class Test extends ApiHelper implements Retry { include("log.xml") include("HostAllocateExtension.xml") include("sdnController.xml") + include("sugonSdnController.xml") + include("TfPortAllocator.xml") } }