Commit 71b35f59 authored by femiadeyemi's avatar femiadeyemi
Browse files

replace the neo4j with mongo db

Motivation:

We need to store media data for some of the entities and neo4j is
not design to store images, video, etc.. Hence, to avoid using
multiple dbs, we sort for a solution that can store the metadata
and the media data.

Modification:

- remove neo4j dependency and add mongo
- change the entities details and re-design it to suit the new db
- adjust the api design
- delete all neo4j relationship pojos
- adjust the tests according to the new definitions

Result:

Cerebrum now uses mongo db.

Target: master
parent 9c3de0bf
Pipeline #74285 failed with stages
in 1 minute and 25 seconds
......@@ -27,7 +27,6 @@
<hibernate-validator.version>6.2.0.Final</hibernate-validator.version>
<dependency-check-maven.version>6.1.6</dependency-check-maven.version>
<dependency-check-maven.cvss-threshold>8</dependency-check-maven.cvss-threshold>
<neo4j-java-driver-spring-boot-starter.version>4.2.4.0</neo4j-java-driver-spring-boot-starter.version>
<nimbus-jose-jwt.version>7.9</nimbus-jose-jwt.version>
<!--suppress UnresolvedMavenProperty -->
<sonar.token>${env.SONAR_AUTH_TOKEN}</sonar.token>
......@@ -46,12 +45,7 @@
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-neo4j</artifactId>
</dependency>
<dependency>
<groupId>org.neo4j.driver</groupId>
<artifactId>neo4j-java-driver-spring-boot-starter</artifactId>
<version>${neo4j-java-driver-spring-boot-starter.version}</version>
<artifactId>spring-boot-starter-data-mongodb</artifactId>
</dependency>
<dependency>
......
......@@ -2,11 +2,11 @@ package de.helmholtz.marketplace.cerebrum;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.data.neo4j.repository.config.EnableNeo4jRepositories;
import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
import org.springframework.transaction.annotation.EnableTransactionManagement;
@EnableTransactionManagement
@EnableNeo4jRepositories
@EnableMongoRepositories
@SpringBootApplication
public class HelmholtzCerebrumApplication
{
......
package de.helmholtz.marketplace.cerebrum.config;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.neo4j.annotation.EnableNeo4jAuditing;
import org.springframework.data.mongodb.config.EnableMongoAuditing;
@Configuration(proxyBeanMethods = false)
@EnableNeo4jAuditing
@EnableMongoAuditing
public class CerebrumDataConfig
{}
package de.helmholtz.marketplace.cerebrum.controller;
import com.fasterxml.jackson.databind.JsonNode;
import com.github.fge.jsonpatch.JsonPatch;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
......@@ -33,8 +34,6 @@ import javax.validation.constraints.Min;
import java.util.List;
import de.helmholtz.marketplace.cerebrum.entity.MarketService;
import de.helmholtz.marketplace.cerebrum.entity.relationship.Management;
import de.helmholtz.marketplace.cerebrum.entity.relationship.ServiceProvider;
import de.helmholtz.marketplace.cerebrum.errorhandling.CerebrumApiError;
import de.helmholtz.marketplace.cerebrum.service.MarketServiceService;
import de.helmholtz.marketplace.cerebrum.utils.CerebrumControllerUtilities;
......@@ -185,53 +184,40 @@ public class MarketServiceController
//serviceProvider
@PreAuthorize("isAuthenticated()")
@Operation(security = @SecurityRequirement(name = "hdf-aai"))
@PostMapping(path = "/providers",
@PostMapping(path = "/{uuid}/provider",
consumes = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<MarketService> addProvider(@Valid @RequestBody ServiceProvider provider)
public ResponseEntity<MarketService> addProvider(@PathVariable("uuid") String uuid,
@Valid @RequestBody JsonNode provider)
{
return marketServiceService.addProvider(provider);
return marketServiceService.addProvider(uuid, provider.get("uuid").asText());
}
@PreAuthorize("isAuthenticated()")
@Operation(security = @SecurityRequirement(name = "hdf-aai"))
@PutMapping(path = "/providers",
consumes = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<MarketService> updateProvider(@Valid @RequestBody ServiceProvider provider)
{
return marketServiceService.updateProvider(provider);
}
@PreAuthorize("isAuthenticated()")
@Operation(security = @SecurityRequirement(name = "hdf-aai"))
@DeleteMapping(path = "/affiliations")
public ResponseEntity<MarketService> deleteProviders(
@RequestParam String serviceKey,
@RequestParam String serviceValue,
@RequestParam String organizationKey,
@RequestParam String organizationValue)
@DeleteMapping(path = "/{uuid}/provider")
public ResponseEntity<MarketService> deleteProviders(@PathVariable("uuid") String uuid,
@Valid @RequestBody JsonNode provider)
{
return marketServiceService.deleteProviders(serviceKey, serviceValue, organizationKey, organizationValue);
return marketServiceService.deleteProvider(uuid, provider.get("uuid").asText());
}
//Management team
@PreAuthorize("isAuthenticated()")
@Operation(security = @SecurityRequirement(name = "hdf-aai"))
@PostMapping(path = "/management",
@PostMapping(path = "/{uuid}/management",
consumes = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<MarketService> addAffiliation(@Valid @RequestBody Management management)
public ResponseEntity<MarketService> addAffiliation(@PathVariable("uuid") String uuid,
@Valid @RequestBody JsonNode teamMember)
{
return marketServiceService.addTeamMember(management);
return marketServiceService.addTeamMember(uuid, teamMember.get("uuid").asText());
}
@PreAuthorize("isAuthenticated()")
@Operation(security = @SecurityRequirement(name = "hdf-aai"))
@DeleteMapping(path = "/management")
public ResponseEntity<MarketService> deleteTeamMember(
@RequestParam String serviceKey,
@RequestParam String serviceValue,
@RequestParam String userKey,
@RequestParam String userValue)
@DeleteMapping(path = "/{uuid}/management")
public ResponseEntity<MarketService> deleteTeamMember(@PathVariable("uuid") String uuid,
@Valid @RequestBody JsonNode teamMember)
{
return marketServiceService.deleteTeamMember(serviceKey, serviceValue, userKey, userValue);
return marketServiceService.deleteTeamMember(uuid, teamMember.get("uuid").asText());
}
}
......@@ -36,9 +36,10 @@ import javax.validation.constraints.NotNull;
import java.util.List;
import de.helmholtz.marketplace.cerebrum.entity.MarketUser;
import de.helmholtz.marketplace.cerebrum.entity.relationship.Affiliation;
import de.helmholtz.marketplace.cerebrum.entity.Person;
import de.helmholtz.marketplace.cerebrum.errorhandling.CerebrumApiError;
import de.helmholtz.marketplace.cerebrum.service.MarketUserService;
import de.helmholtz.marketplace.cerebrum.service.PersonService;
import de.helmholtz.marketplace.cerebrum.utils.CerebrumControllerUtilities;
@RestController
......@@ -49,12 +50,15 @@ import de.helmholtz.marketplace.cerebrum.utils.CerebrumControllerUtilities;
public class MarketUserController {
private final WebClient authorisationServer;
private final MarketUserService marketUserService;
private final PersonService personService;
public MarketUserController(WebClient authorisationServer,
MarketUserService marketUserService)
MarketUserService marketUserService,
PersonService personService)
{
this.authorisationServer = authorisationServer;
this.marketUserService = marketUserService;
this.personService = personService;
}
@PreAuthorize("isAuthenticated()")
......@@ -235,24 +239,21 @@ public class MarketUserController {
*/
@PreAuthorize("isAuthenticated()")
@Operation(security = @SecurityRequirement(name = "hdf-aai"))
@PostMapping(path = "/affiliations",
@PostMapping(path = "/{uuid}/affiliation",
consumes = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<MarketUser> addAffiliation(@Valid @RequestBody Affiliation affiliation)
public ResponseEntity<MarketUser> addAffiliation(@PathVariable(name = "uuid") String uuid,
@Valid @RequestBody JsonNode affiliation)
{
return marketUserService.addAffiliations(affiliation);
return marketUserService.addAffiliation(uuid, affiliation.get("uuid").asText());
}
@PreAuthorize("isAuthenticated()")
@Operation(security = @SecurityRequirement(name = "hdf-aai"))
@DeleteMapping(path = "/affiliations")
public ResponseEntity<MarketUser> deleteAffiliations(
@RequestParam String userKey,
@RequestParam String userValue,
@RequestParam String organizationKey,
@RequestParam String organizationValue)
@DeleteMapping(path = "/{uuid}/affiliation")
public ResponseEntity<MarketUser> deleteAffiliation(@PathVariable(name = "uuid") String uuid,
@Valid @RequestBody JsonNode affiliation)
{
return marketUserService.deleteAffiliations(
userKey, userValue, organizationKey, organizationValue);
return marketUserService.deleteAffiliation(uuid, affiliation.get("uuid").asText());
}
private MarketUser checkAndAdd(@NotNull JsonNode user)
......@@ -260,10 +261,18 @@ public class MarketUserController {
MarketUser knownUser = marketUserService.getUser(user);
if (knownUser == null) {
MarketUser newUser = new MarketUser();
newUser.setFirstName(user.get("given_name").asText());
newUser.setLastName(user.get("family_name").asText());
Person p = personService.getPerson(user.get("given_name").asText(), user.get("family_name").asText());
if (p == null) {
Person newPerson = new Person();
newPerson.setFirstName(user.get("given_name").asText());
newPerson.setLastName(user.get("family_name").asText());
newPerson.addEmail(user.get("email").asText());
Person details = personService.createPerson(newPerson);
newUser.setProfile(details);
} else {
newUser.setProfile(p);
}
newUser.setSub(user.get("sub").asText());
newUser.setEmail(user.get("email").asText());
return marketUserService.createUser(newUser);
}
return knownUser;
......
......@@ -32,6 +32,8 @@ import javax.validation.Valid;
import javax.validation.constraints.Min;
import java.util.List;
import de.helmholtz.marketplace.cerebrum.entity.MarketService;
import de.helmholtz.marketplace.cerebrum.entity.MarketUser;
import de.helmholtz.marketplace.cerebrum.entity.Organization;
import de.helmholtz.marketplace.cerebrum.errorhandling.CerebrumApiError;
import de.helmholtz.marketplace.cerebrum.service.OrganizationService;
......@@ -190,4 +192,60 @@ public class OrganizationController {
{
return organizationService.deleteOrganisation(uuid);
}
/* get list of hosted services */
@PreAuthorize("isAuthenticated()")
@Operation(summary = "get list of hosted services",
description = "A list of Services which are provided by the organization",
security = @SecurityRequirement(name = "hdf-aai"))
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "successful operation",
content = @Content(array = @ArraySchema(
schema = @Schema(implementation = MarketService.class)))),
@ApiResponse(responseCode = "401", description = "unauthorised", content = @Content())
})
@GetMapping(path = "/{uuid}/hosted-services")
public Iterable<MarketService> listHostedServices(
@PathVariable(name = "uuid") String uuid,
@Parameter(description = "specify the page number")
@RequestParam(value = "page", defaultValue = "0") @Min(0) Integer page,
@Parameter(description = "limit the number of records returned in one page")
@RequestParam(value = "size", defaultValue = "20") @Min(1) Integer size,
@Parameter(description = "sort the fetched data in either ascending (asc) " +
"or descending (desc) according to one or more of the service " +
"properties. Eg. to sort the list in ascending order base on the " +
"name property; the value will be set to name.asc")
@RequestParam(value = "sort", defaultValue = "name.asc") List<String> sorts)
{
return organizationService.getHostedServices(
uuid, PageRequest.of(page, size, Sort.by(CerebrumControllerUtilities.getOrders(sorts))));
}
/* get list of members */
@PreAuthorize("isAuthenticated()")
@Operation(summary = "get list of hosted services",
description = "List of people that are affiliated with this organisation",
security = @SecurityRequirement(name = "hdf-aai"))
@ApiResponses(value = {
@ApiResponse(responseCode = "200", description = "successful operation",
content = @Content(array = @ArraySchema(
schema = @Schema(implementation = MarketUser.class)))),
@ApiResponse(responseCode = "401", description = "unauthorised", content = @Content())
})
@GetMapping(path = "/{uuid}/members")
public Iterable<MarketUser> listMembers(
@PathVariable(name = "uuid") String uuid,
@Parameter(description = "specify the page number")
@RequestParam(value = "page", defaultValue = "0") @Min(0) Integer page,
@Parameter(description = "limit the number of records returned in one page")
@RequestParam(value = "size", defaultValue = "20") @Min(1) Integer size,
@Parameter(description = "sort the fetched data in either ascending (asc) " +
"or descending (desc) according to one or more of the user " +
"properties. Eg. to sort the list in ascending order base on the " +
"profile property; the value will be set to profile.firstName.asc")
@RequestParam(value = "sort", defaultValue = "profile.firstName.asc") List<String> sorts)
{
return organizationService.listKnownMembers(uuid,
PageRequest.of(page, size, Sort.by(CerebrumControllerUtilities.getOrders(sorts))));
}
}
package de.helmholtz.marketplace.cerebrum.entity;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.Setter;
import org.neo4j.ogm.annotation.GeneratedValue;
import org.neo4j.ogm.annotation.Id;
import org.neo4j.ogm.annotation.NodeEntity;
import org.neo4j.ogm.annotation.Relationship;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.DBRef;
import org.springframework.data.mongodb.core.mapping.Document;
import org.springframework.lang.Nullable;
import javax.validation.constraints.Email;
import javax.validation.constraints.NotNull;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.TreeSet;
import de.helmholtz.marketplace.cerebrum.entity.relationship.Management;
import de.helmholtz.marketplace.cerebrum.entity.relationship.ServiceProvider;
import de.helmholtz.marketplace.cerebrum.utils.CerebrumEntityUuidGenerator;
import static de.helmholtz.marketplace.cerebrum.utils.CerebrumEntityUuidGenerator.generate;
import static org.neo4j.ogm.annotation.Relationship.INCOMING;
@Setter(AccessLevel.PUBLIC)
@Getter(AccessLevel.PUBLIC)
@NodeEntity
@Document
public class MarketService extends AuditMetadata
{
@Schema(description = "Unique identifier of the market service.",
example = "svc-01eac6d7-0d35-1812-a3ed-24aec4231940", required = true)
@Setter(AccessLevel.NONE)
@Id @GeneratedValue(strategy = CerebrumEntityUuidGenerator.class)
private String uuid;
@Id
private String uuid = generate("svc");
@NotNull
@Schema(description = "Name of a Service", example = "Sync+Share", required = true)
......@@ -71,25 +69,64 @@ public class MarketService extends AuditMetadata
private Phase phase;
@Schema(description = "")
private List<String> targetGroup = new ArrayList<>();
private Set<String> targetGroup = new TreeSet<>();
@Schema(description = "")
private List<String> tags = new ArrayList<>();
private Set<String> tags = new TreeSet<>();
@Schema(description = "List of services provided by this organisation")
@JsonIgnoreProperties({"marketService"})
@Relationship(type = "HOSTED_BY")
private List<ServiceProvider> serviceProviders;
@DBRef
private List<Organization> serviceProviders = new ArrayList<>();
@JsonIgnoreProperties({"marketService"})
@Relationship(type = "MANAGES", direction = INCOMING)
private List<Management> managementTeam;
@Schema(description = "")
@DBRef
private List<Person> managementTeam = new ArrayList<>();
public void setUuid(String uuid)
public void setUuid(@Nullable String uuid)
{
this.uuid = Boolean.TRUE.equals(
CerebrumEntityUuidGenerator.isValid(uuid))
? uuid : generate("org");
? uuid : generate("svc");
}
public void addTarget(String target)
{
targetGroup.add(target);
}
public void removeTarget(String target)
{
targetGroup.remove(target);
}
public void addTag(String tag)
{
tags.add(tag);
}
public void removeTag(String tag)
{
tags.remove(tag);
}
public void addProvider(Organization org)
{
serviceProviders.add(org);
}
public void removeProvider(Organization org)
{
serviceProviders.remove(org);
}
public void addTeamMember(Person member)
{
managementTeam.add(member);
}
public void removeTeamMember(Person member)
{
managementTeam.remove(member);
}
@Override
......
package de.helmholtz.marketplace.cerebrum.entity;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.Setter;
import org.neo4j.ogm.annotation.NodeEntity;
import org.neo4j.ogm.annotation.Relationship;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.DBRef;
import org.springframework.data.mongodb.core.mapping.Document;
import org.springframework.lang.Nullable;
import javax.validation.constraints.Size;
import java.util.ArrayList;
import java.util.List;
import de.helmholtz.marketplace.cerebrum.entity.relationship.Affiliation;
import de.helmholtz.marketplace.cerebrum.entity.relationship.Management;
import de.helmholtz.marketplace.cerebrum.utils.CerebrumEntityUuidGenerator;
import static de.helmholtz.marketplace.cerebrum.utils.CerebrumEntityUuidGenerator.generate;
@Setter(AccessLevel.PUBLIC)
@Getter(AccessLevel.PUBLIC)
@NodeEntity
public class MarketUser extends Person
@Document
public class MarketUser extends AuditMetadata
{
@Schema(description = "Unique identifier of the market service.",
example = "svc-01eac6d7-0d35-1812-a3ed-24aec4231940", required = true)
@Setter(AccessLevel.NONE)
@Id
private String uuid = generate("usr");
@Schema(description = "User chosen name to represent him or herself", example = "pm")
@Size(max = 20)
private String screenName;
......@@ -27,11 +36,32 @@ public class MarketUser extends Person
example = "110248495921238986420", required = true)
private String sub;
@JsonIgnoreProperties("user")
@Relationship(type = "BELONGS_TO")
private List<Affiliation> affiliations;
@Schema(description = "Helmholtz AAI generated unique user identifier")
@DBRef
private Person profile;
@Schema(description = "")
@DBRef
private List<Organization> affiliations = new ArrayList<>();
@Schema(description = "")
@DBRef
private List<MarketService> managedServices = new ArrayList<>();
public void addAffiliation(Organization org)
{
affiliations.add(org);
}
public void removeAffiliation(Organization org)
{
affiliations.remove(org);
}
@JsonIgnoreProperties("marketUser")
@Relationship(type = "MANAGES")
private List<Management> managedServices;
public void setUuid(@Nullable String uuid)
{
this.uuid = Boolean.TRUE.equals(
CerebrumEntityUuidGenerator.isValid(uuid))
? uuid : generate("usr");
}
}
package de.helmholtz.marketplace.cerebrum.entity;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.Setter;
import org.hibernate.validator.constraints.URL;
import org.neo4j.ogm.annotation.GeneratedValue;
import org.neo4j.ogm.annotation.Id;
import org.neo4j.ogm.annotation.NodeEntity;
import org.neo4j.ogm.annotation.Relationship;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import javax.validation.constraints.NotNull;
import java.util.List;
import java.util.Objects;
import de.helmholtz.marketplace.cerebrum.entity.relationship.Affiliation;
import de.helmholtz.marketplace.cerebrum.entity.relationship.ServiceProvider;
import de.helmholtz.marketplace.cerebrum.utils.CerebrumEntityUuidGenerator;
import static de.helmholtz.marketplace.cerebrum.utils.CerebrumEntityUuidGenerator.generate;
import static org.neo4j.ogm.annotation.Relationship.INCOMING;
@Schema(name = "Organization", description = "POJO that represents a single organization entry.")
@Setter(AccessLevel.PUBLIC)
@Getter(AccessLevel.PUBLIC)
@NodeEntity
@Document
public class Organization
{
@Schema(description = "Unique identifier of the organisation",
example = "org-01eac6d7-0d35-1812-a3ed-24aec4231940", required = true)
@Setter(AccessLevel.NONE)
@Id @GeneratedValue(strategy = CerebrumEntityUuidGenerator.class)
private String uuid;
@Id
private String uuid = generate("org");
@Schema(description = "Name of the organisation in full",
example = "Deutsches Elektronen-Synchrotron", required = true)
......@@ -62,16 +55,6 @@ public class Organization
@Schema(description = "", example = "HELMHOLTZ_CENTRE")
private Type type;
@JsonIgnoreProperties({"organization"})
@Schema(description = "A list of Services which are provided by the organization")
@Relationship(type = "HOSTED_BY", direction = INCOMING)
private List<ServiceProvider> hostedServices;
@JsonIgnoreProperties("organization")
@Schema(description = "List of people that are affiliated with this organisation")
@Relationship(type = "BELONGS_TO", direction = INCOMING)
private List<Affiliation> members;
public void setUuid(String uuid)
{
this.uuid = Boolean.TRUE.equals(
......
......@@ -4,12 +4,14 @@ import io.swagger.v3.oas.annotations.media.Schema;
import lombok.AccessLevel;
import lombok.Getter;
import lombok.Setter;
import org.neo4j.ogm.annotation.GeneratedValue;
import org.neo4j.ogm.annotation.Id;
import org.springframework.data.annotation.Id;
import org.springframework.data.mongodb.core.mapping.Document;
import javax.validation.constraints.Email;